Anurag1734/cuda-error-resolution-analysis
07
1[2 {3 "post_stream": {4 "posts": [5 {6 "id": 251742,7 "name": "Gadiraju sanjay varma",8 "username": "Sanjayvarma11",9 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png",10 "created_at": "2020-12-16T06:56:37.175Z",11 "cooked": "<p><a class=\"mention\" href=\"/u/ptrblck\">@ptrblck</a><br>\n<a class=\"mention\" href=\"/u/smth\">@smth</a></p>\n<p>Hello everyone.We are trying to implement distributed computing using pytorch DistributedDataParallel.<br>\nWe have two computers connected via LAN and we are trying to Distribute computation.</p>\n<p><strong>Here is a script which i am running on server:</strong></p>\n<pre><code class=\"lang-auto\"> \nimport os\nfrom datetime import datetime\nimport argparse\nimport torch.multiprocessing as mp\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport torch.distributed as dist\nfrom torch.nn.parallel import DistributedDataParallel as DDP\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('-n', '--nodes', default=2, type=int, metavar='N',\n help='number of data loading workers (default: 4)')\n parser.add_argument('-g', '--gpus', default=1, type=int,\n help='number of gpus per node')\n parser.add_argument('-nr', '--nr', default=0, type=int,\n help='ranking within the nodes')\n parser.add_argument('--epochs', default=2, type=int, metavar='N',\n help='number of total epochs to run')\n args = parser.parse_args()\n args.world_size = args.gpus * args.nodes\n \n os.environ['MASTER_ADDR'] = <serverIP>\n os.environ['MASTER_PORT'] = <Port>\n mp.spawn(train, nprocs=args.gpus, args=(args,))\n\n\nclass ConvNet(nn.Module):\n def __init__(self, num_classes=10):\n super(ConvNet, self).__init__()\n self.layer1 = nn.Sequential(\n nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(16),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.layer2 = nn.Sequential(\n nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(32),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.fc = nn.Linear(7*7*32, num_classes)\n\n def forward(self, x):\n out = self.layer1(x)\n out = self.layer2(out)\n out = out.reshape(out.size(0), -1)\n out = self.fc(out)\n return out\n\n\ndef train(gpu, args):\n rank = args.nr * args.gpus + gpu\n dist.init_process_group(backend='gloo', init_method='file:\\\\C:\\\\Users\\\\VIT\\\\Desktop\\\\test\\\\glooBackened.py', world_size=args.world_size, rank=rank)\n torch.manual_seed(0)\n model = ConvNet()\n torch.cuda.set_device(gpu)\n model.cuda(gpu)\n batch_size = 100\n # define loss function (criterion) and optimizer\n criterion = nn.CrossEntropyLoss().cuda(gpu)\n optimizer = torch.optim.SGD(model.parameters(), 1e-4)\n # Wrap the model\n model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])\n # Data loading code\n train_dataset = torchvision.datasets.MNIST(root='./data',\n train=True,\n transform=transforms.ToTensor(),\n download=True)\n train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,\n num_replicas=args.world_size,\n rank=rank)\n train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size,\n shuffle=False,\n num_workers=0,\n pin_memory=True,\n sampler=train_sampler)\n\n start = datetime.now()\n total_step = len(train_loader)\n for epoch in range(args.epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.cuda(non_blocking=True)\n labels = labels.cuda(non_blocking=True)\n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n\n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n if (i + 1) % 100 == 0 and gpu == 0:\n print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch + 1, args.epochs, i + 1, total_step,\n loss.item()))\n if gpu == 0:\n print(\"Training complete in: \" + str(datetime.now() - start))\n\n\nif __name__ == '__main__':\n main()\n\n</code></pre>\n<p>Here if we give nodes as 1 then it is executing perfectly.</p>\n<p><strong>Here is a script which i am running on client:</strong></p>\n<pre><code class=\"lang-auto\"> \nimport os\nfrom datetime import datetime\nimport argparse\nimport torch.multiprocessing as mp\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport torch.distributed as dist\nfrom torch.nn.parallel import DistributedDataParallel as DDP\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('-n', '--nodes', default=2, type=int, metavar='N',\n help='number of data loading workers (default: 4)')\n parser.add_argument('-g', '--gpus', default=1, type=int,\n help='number of gpus per node')\n parser.add_argument('-nr', '--nr', default=0, type=int,\n help='ranking within the nodes')\n parser.add_argument('--epochs', default=2, type=int, metavar='N',\n help='number of total epochs to run')\n args = parser.parse_args()\n args.world_size = args.gpus * args.nodes\n \n os.environ['MASTER_ADDR'] = <serverIP>\n os.environ['MASTER_PORT'] = <Port>\n mp.spawn(train, nprocs=args.gpus, args=(args,))\n\n\nclass ConvNet(nn.Module):\n def __init__(self, num_classes=10):\n super(ConvNet, self).__init__()\n self.layer1 = nn.Sequential(\n nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(16),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.layer2 = nn.Sequential(\n nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(32),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.fc = nn.Linear(7*7*32, num_classes)\n\n def forward(self, x):\n out = self.layer1(x)\n out = self.layer2(out)\n out = out.reshape(out.size(0), -1)\n out = self.fc(out)\n return out\n\n\ndef train(gpu, args):\n rank = args.nr * args.gpus + gpu\n dist.init_process_group(backend='gloo', init_method='file:\\\\C:\\\\Users\\\\VIT\\\\Desktop\\\\test\\\\glooBackened.py', world_size=args.world_size, rank=rank)\n torch.manual_seed(0)\n model = ConvNet()\n torch.cuda.set_device(gpu)\n model.cuda(gpu)\n batch_size = 100\n # define loss function (criterion) and optimizer\n criterion = nn.CrossEntropyLoss().cuda(gpu)\n optimizer = torch.optim.SGD(model.parameters(), 1e-4)\n # Wrap the model\n model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])\n # Data loading code\n train_dataset = torchvision.datasets.MNIST(root='./data',\n train=True,\n transform=transforms.ToTensor(),\n download=True)\n train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,\n num_replicas=args.world_size,\n rank=rank)\n train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size,\n shuffle=False,\n num_workers=0,\n pin_memory=True,\n sampler=train_sampler)\n\n start = datetime.now()\n total_step = len(train_loader)\n for epoch in range(args.epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.cuda(non_blocking=True)\n labels = labels.cuda(non_blocking=True)\n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n\n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n if (i + 1) % 100 == 0 and gpu == 0:\n print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch + 1, args.epochs, i + 1, total_step,\n loss.item()))\n if gpu == 0:\n print(\"Training complete in: \" + str(datetime.now() - start))\n\n\nif __name__ == '__main__':\n main()\n\n</code></pre>\n<p>According to article we read we are having same script and the only change is the rank.<br>\nWe are setting rank 0 for server and rank 1 for client.Both of them are waiting for each other and not running at all.</p>\n<p>I tried ping in Windows commandPrompt and it is working fine.<br>\nI also tried to run both server and client script on same system by keeping ip address as localhost.Still it is waiting.</p>\n<p>Hoping that someone will solve our proble.Thank you</p>",12 "post_number": 1,13 "post_type": 1,14 "posts_count": 6,15 "updated_at": "2020-12-16T07:02:15.693Z",16 "reply_count": 0,17 "reply_to_post_number": null,18 "quote_count": 0,19 "incoming_link_count": 289,20 "reads": 22,21 "readers_count": 21,22 "score": 1449.4,23 "yours": false,24 "topic_id": 106281,25 "topic_slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",26 "display_username": "Gadiraju sanjay varma",27 "primary_group_name": null,28 "flair_name": null,29 "flair_url": null,30 "flair_bg_color": null,31 "flair_color": null,32 "flair_group_id": null,33 "badges_granted": [],34 "version": 2,35 "can_edit": false,36 "can_delete": false,37 "can_recover": false,38 "can_see_hidden_post": false,39 "can_wiki": false,40 "read": true,41 "user_title": null,42 "bookmarked": false,43 "actions_summary": [],44 "moderator": false,45 "admin": false,46 "staff": false,47 "user_id": 26932,48 "hidden": false,49 "trust_level": 2,50 "deleted_at": null,51 "user_deleted": false,52 "edit_reason": null,53 "can_view_edit_history": true,54 "wiki": false,55 "post_url": "/t/having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients/106281/1",56 "can_accept_answer": false,57 "can_unaccept_answer": false,58 "accepted_answer": false,59 "topic_accepted_answer": null,60 "can_vote": false61 },62 {63 "id": 251757,64 "name": "",65 "username": "ptrblck",66 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",67 "created_at": "2020-12-16T07:48:55.498Z",68 "cooked": "<p>Please don’t tag specific users, as it might discourage others to post an answer and you might tag a non-expert on this topic.</p>",69 "post_number": 2,70 "post_type": 1,71 "posts_count": 6,72 "updated_at": "2020-12-16T07:48:55.498Z",73 "reply_count": 1,74 "reply_to_post_number": null,75 "quote_count": 0,76 "incoming_link_count": 4,77 "reads": 17,78 "readers_count": 16,79 "score": 28.4,80 "yours": false,81 "topic_id": 106281,82 "topic_slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",83 "display_username": "",84 "primary_group_name": null,85 "flair_name": null,86 "flair_url": null,87 "flair_bg_color": null,88 "flair_color": null,89 "flair_group_id": null,90 "badges_granted": [],91 "version": 1,92 "can_edit": false,93 "can_delete": false,94 "can_recover": false,95 "can_see_hidden_post": false,96 "can_wiki": false,97 "read": true,98 "user_title": "",99 "bookmarked": false,100 "actions_summary": [],101 "moderator": true,102 "admin": true,103 "staff": true,104 "user_id": 3534,105 "hidden": false,106 "trust_level": 2,107 "deleted_at": null,108 "user_deleted": false,109 "edit_reason": null,110 "can_view_edit_history": true,111 "wiki": false,112 "post_url": "/t/having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients/106281/2",113 "can_accept_answer": false,114 "can_unaccept_answer": false,115 "accepted_answer": false,116 "topic_accepted_answer": null117 },118 {119 "id": 251942,120 "name": "Rohan Varma",121 "username": "rvarm1",122 "avatar_template": "/user_avatar/discuss.pytorch.org/rvarm1/{size}/15821_2.png",123 "created_at": "2020-12-16T20:22:57.379Z",124 "cooked": "<p>The nodes become aware of each other through a process called rendezvous, which happens within <code>dist.init_process_group</code> which is a synchronization point for all nodes. Looking at the arguments you’ve passed into <code>init_process_group</code>, if your client is on a different machine and your filesystem is not somehow networked, different files will be used for initialization, so the processes will never come to know about each other.</p>\n<p>If you are using windows for DDP training, we only support file-backed initiliazation and single-machine use cases. We have landed TCP-based initialization support in PyTorch master (<a href=\"https://github.com/pytorch/pytorch/pull/47749\" rel=\"noopener nofollow ugc\">https://github.com/pytorch/pytorch/pull/47749</a>), and you can find docs to use TCP-based init here: <a href=\"https://pytorch.org/docs/stable/distributed.html#tcp-initialization\" rel=\"noopener nofollow ugc\">https://pytorch.org/docs/stable/distributed.html#tcp-initialization</a></p>",125 "post_number": 3,126 "post_type": 1,127 "posts_count": 6,128 "updated_at": "2020-12-16T20:22:57.379Z",129 "reply_count": 2,130 "reply_to_post_number": null,131 "quote_count": 0,132 "incoming_link_count": 2,133 "reads": 17,134 "readers_count": 16,135 "score": 23.4,136 "yours": false,137 "topic_id": 106281,138 "topic_slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",139 "display_username": "Rohan Varma",140 "primary_group_name": null,141 "flair_name": null,142 "flair_url": null,143 "flair_bg_color": null,144 "flair_color": null,145 "flair_group_id": null,146 "badges_granted": [],147 "version": 1,148 "can_edit": false,149 "can_delete": false,150 "can_recover": false,151 "can_see_hidden_post": false,152 "can_wiki": false,153 "link_counts": [154 {155 "url": "https://github.com/pytorch/pytorch/pull/47749",156 "internal": false,157 "reflection": false,158 "clicks": 6159 },160 {161 "url": "https://pytorch.org/docs/stable/distributed.html#tcp-initialization",162 "internal": false,163 "reflection": false,164 "title": "Distributed communication package - torch.distributed — PyTorch 1.7.0 documentation",165 "clicks": 5166 }167 ],168 "read": true,169 "user_title": "",170 "bookmarked": false,171 "actions_summary": [],172 "moderator": false,173 "admin": false,174 "staff": false,175 "user_id": 22425,176 "hidden": false,177 "trust_level": 2,178 "deleted_at": null,179 "user_deleted": false,180 "edit_reason": null,181 "can_view_edit_history": true,182 "wiki": false,183 "post_url": "/t/having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients/106281/3",184 "can_accept_answer": false,185 "can_unaccept_answer": false,186 "accepted_answer": false,187 "topic_accepted_answer": null188 },189 {190 "id": 252003,191 "name": "Gadiraju sanjay varma",192 "username": "Sanjayvarma11",193 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png",194 "created_at": "2020-12-17T03:11:39.255Z",195 "cooked": "<p>Sorry sir.i will never do it again.</p>",196 "post_number": 4,197 "post_type": 1,198 "posts_count": 6,199 "updated_at": "2020-12-17T03:11:39.255Z",200 "reply_count": 0,201 "reply_to_post_number": 2,202 "quote_count": 0,203 "incoming_link_count": 1,204 "reads": 16,205 "readers_count": 15,206 "score": 8.2,207 "yours": false,208 "topic_id": 106281,209 "topic_slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",210 "display_username": "Gadiraju sanjay varma",211 "primary_group_name": null,212 "flair_name": null,213 "flair_url": null,214 "flair_bg_color": null,215 "flair_color": null,216 "flair_group_id": null,217 "badges_granted": [],218 "version": 1,219 "can_edit": false,220 "can_delete": false,221 "can_recover": false,222 "can_see_hidden_post": false,223 "can_wiki": false,224 "read": true,225 "user_title": null,226 "reply_to_user": {227 "id": 3534,228 "username": "ptrblck",229 "name": "",230 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"231 },232 "bookmarked": false,233 "actions_summary": [],234 "moderator": false,235 "admin": false,236 "staff": false,237 "user_id": 26932,238 "hidden": false,239 "trust_level": 2,240 "deleted_at": null,241 "user_deleted": false,242 "edit_reason": null,243 "can_view_edit_history": true,244 "wiki": false,245 "post_url": "/t/having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients/106281/4",246 "can_accept_answer": false,247 "can_unaccept_answer": false,248 "accepted_answer": false,249 "topic_accepted_answer": null250 },251 {252 "id": 252004,253 "name": "Gadiraju sanjay varma",254 "username": "Sanjayvarma11",255 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png",256 "created_at": "2020-12-17T03:12:02.297Z",257 "cooked": "<p>Thnak you sir for replying.we will try it out</p>",258 "post_number": 5,259 "post_type": 1,260 "posts_count": 6,261 "updated_at": "2020-12-17T03:12:02.297Z",262 "reply_count": 0,263 "reply_to_post_number": 3,264 "quote_count": 0,265 "incoming_link_count": 3,266 "reads": 16,267 "readers_count": 15,268 "score": 18.2,269 "yours": false,270 "topic_id": 106281,271 "topic_slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",272 "display_username": "Gadiraju sanjay varma",273 "primary_group_name": null,274 "flair_name": null,275 "flair_url": null,276 "flair_bg_color": null,277 "flair_color": null,278 "flair_group_id": null,279 "badges_granted": [],280 "version": 1,281 "can_edit": false,282 "can_delete": false,283 "can_recover": false,284 "can_see_hidden_post": false,285 "can_wiki": false,286 "read": true,287 "user_title": null,288 "reply_to_user": {289 "id": 22425,290 "username": "rvarm1",291 "name": "Rohan Varma",292 "avatar_template": "/user_avatar/discuss.pytorch.org/rvarm1/{size}/15821_2.png"293 },294 "bookmarked": false,295 "actions_summary": [],296 "moderator": false,297 "admin": false,298 "staff": false,299 "user_id": 26932,300 "hidden": false,301 "trust_level": 2,302 "deleted_at": null,303 "user_deleted": false,304 "edit_reason": null,305 "can_view_edit_history": true,306 "wiki": false,307 "post_url": "/t/having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients/106281/5",308 "can_accept_answer": false,309 "can_unaccept_answer": false,310 "accepted_answer": false,311 "topic_accepted_answer": null312 },313 {314 "id": 252018,315 "name": "Gadiraju sanjay varma",316 "username": "Sanjayvarma11",317 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png",318 "created_at": "2020-12-17T04:43:06.552Z",319 "cooked": "<p>Sir i tried using Tcp but it is giving me deprecated error.</p>\n<p><strong>Code for server is as follows</strong></p>\n<pre><code class=\"lang-auto\"> \nimport os\nfrom datetime import datetime\nimport argparse\nimport torch.multiprocessing as mp\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch\nimport torch.nn as nn\nimport torch.distributed as dist\nfrom torch.nn.parallel import DistributedDataParallel as DDP\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('-n', '--nodes', default=2, type=int, metavar='N',\n help='number of data loading workers (default: 4)')\n parser.add_argument('-g', '--gpus', default=1, type=int,\n help='number of gpus per node')\n parser.add_argument('-nr', '--nr', default=0, type=int,\n help='ranking within the nodes')\n parser.add_argument('--epochs', default=2, type=int, metavar='N',\n help='number of total epochs to run')\n args = parser.parse_args()\n args.world_size = args.gpus * args.nodes\n \n os.environ['MASTER_ADDR'] = '10.0.45.47'\n os.environ['MASTER_PORT'] = '8888'\n torch.cuda.set_device(0)\n mp.spawn(train, nprocs=args.gpus, args=(args,))\n\n\nclass ConvNet(nn.Module):\n def __init__(self, num_classes=10):\n super(ConvNet, self).__init__()\n self.layer1 = nn.Sequential(\n nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(16),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.layer2 = nn.Sequential(\n nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),\n nn.BatchNorm2d(32),\n nn.ReLU(),\n nn.MaxPool2d(kernel_size=2, stride=2))\n self.fc = nn.Linear(7*7*32, num_classes)\n\n def forward(self, x):\n out = self.layer1(x)\n out = self.layer2(out)\n out = out.reshape(out.size(0), -1)\n out = self.fc(out)\n return out\n\n\ndef train(gpu, args):\n rank = args.nr * args.gpus + gpu\n dist.init_process_group(backend='tcp', init_method='tcp://10.0.45.47:8888', world_size=args.world_size, rank=rank)\n torch.manual_seed(0)\n model = ConvNet()\n torch.cuda.set_device(gpu)\n model.cuda(gpu)\n batch_size = 100\n # define loss function (criterion) and optimizer\n criterion = nn.CrossEntropyLoss().cuda(gpu)\n optimizer = torch.optim.SGD(model.parameters(), 1e-4)\n # Wrap the model\n model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])\n # Data loading code\n train_dataset = torchvision.datasets.MNIST(root='./data',\n train=True,\n transform=transforms.ToTensor(),\n download=True)\n train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,\n num_replicas=args.world_size,\n rank=rank)\n train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n batch_size=batch_size,\n shuffle=False,\n num_workers=0,\n pin_memory=True,\n sampler=train_sampler)\n\n start = datetime.now()\n total_step = len(train_loader)\n for epoch in range(args.epochs):\n for i, (images, labels) in enumerate(train_loader):\n images = images.cuda(non_blocking=True)\n labels = labels.cuda(non_blocking=True)\n # Forward pass\n outputs = model(images)\n loss = criterion(outputs, labels)\n\n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n if (i + 1) % 100 == 0 and gpu == 0:\n print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch + 1, args.epochs, i + 1, total_step,\n loss.item()))\n if gpu == 0:\n print(\"Training complete in: \" + str(datetime.now() - start))\n\n\nif __name__ == '__main__':\n main()\n\n</code></pre>\n<p><strong>Error is as follows:</strong></p>\n<pre><code class=\"lang-auto\"> raise ValueError(\"TCP backend has been deprecated. Please use \"\nValueError: TCP backend has been deprecated. Please use Gloo or MPI backend for collective operations on CPU tensors.\n</code></pre>\n<p>Thank you sir.</p>",320 "post_number": 6,321 "post_type": 1,322 "posts_count": 6,323 "updated_at": "2020-12-17T04:43:06.552Z",324 "reply_count": 0,325 "reply_to_post_number": 3,326 "quote_count": 0,327 "incoming_link_count": 12,328 "reads": 16,329 "readers_count": 15,330 "score": 63.2,331 "yours": false,332 "topic_id": 106281,333 "topic_slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",334 "display_username": "Gadiraju sanjay varma",335 "primary_group_name": null,336 "flair_name": null,337 "flair_url": null,338 "flair_bg_color": null,339 "flair_color": null,340 "flair_group_id": null,341 "badges_granted": [],342 "version": 1,343 "can_edit": false,344 "can_delete": false,345 "can_recover": false,346 "can_see_hidden_post": false,347 "can_wiki": false,348 "read": true,349 "user_title": null,350 "reply_to_user": {351 "id": 22425,352 "username": "rvarm1",353 "name": "Rohan Varma",354 "avatar_template": "/user_avatar/discuss.pytorch.org/rvarm1/{size}/15821_2.png"355 },356 "bookmarked": false,357 "actions_summary": [],358 "moderator": false,359 "admin": false,360 "staff": false,361 "user_id": 26932,362 "hidden": false,363 "trust_level": 2,364 "deleted_at": null,365 "user_deleted": false,366 "edit_reason": null,367 "can_view_edit_history": true,368 "wiki": false,369 "post_url": "/t/having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients/106281/6",370 "can_accept_answer": false,371 "can_unaccept_answer": false,372 "accepted_answer": false,373 "topic_accepted_answer": null374 }375 ],376 "stream": [377 251742,378 251757,379 251942,380 252003,381 252004,382 252018383 ]384 },385 "timeline_lookup": [386 [387 1,388 1775389 ],390 [391 3,392 1774393 ]394 ],395 "suggested_topics": [396 {397 "fancy_title": "Embedded Python can’t import torch in a C++ project",398 "id": 212740,399 "title": "Embedded Python can't import torch in a C++ project",400 "slug": "embedded-python-cant-import-torch-in-a-c-project",401 "posts_count": 2,402 "reply_count": 0,403 "highest_post_number": 2,404 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/2/2/22a79ccd269f479f2e40ca8fff2c77af809bccb9_2_1024x383.png",405 "created_at": "2024-11-09T13:44:00.261Z",406 "last_posted_at": "2024-12-01T05:37:43.874Z",407 "bumped": true,408 "bumped_at": "2024-12-01T05:37:43.874Z",409 "archetype": "regular",410 "unseen": false,411 "pinned": false,412 "unpinned": null,413 "visible": true,414 "closed": false,415 "archived": false,416 "bookmarked": null,417 "liked": null,418 "tags_descriptions": {},419 "like_count": 1,420 "views": 114,421 "category_id": 20,422 "featured_link": null,423 "has_accepted_answer": true,424 "posters": [425 {426 "extras": "latest single",427 "description": "Original Poster, Most Recent Poster, Accepted Answer",428 "user": {429 "id": 80779,430 "username": "Himateja_Nallani",431 "name": "Himateja Nallani",432 "avatar_template": "/user_avatar/discuss.pytorch.org/himateja_nallani/{size}/73874_2.png",433 "trust_level": 0434 }435 }436 ]437 },438 {439 "fancy_title": "Connect [127.0.1.1]:20892: Connection refused",440 "id": 212976,441 "title": "Connect [127.0.1.1]:20892: Connection refused",442 "slug": "connect-127-0-1-1-connection-refused",443 "posts_count": 1,444 "reply_count": 0,445 "highest_post_number": 1,446 "image_url": null,447 "created_at": "2024-11-14T09:15:49.089Z",448 "last_posted_at": "2024-11-14T09:15:49.168Z",449 "bumped": true,450 "bumped_at": "2024-11-14T09:15:49.168Z",451 "archetype": "regular",452 "unseen": false,453 "pinned": false,454 "unpinned": null,455 "visible": true,456 "closed": false,457 "archived": false,458 "bookmarked": null,459 "liked": null,460 "unicode_title": "Connect [127.0.1.1]:20892: Connection refused",461 "tags_descriptions": {},462 "like_count": 0,463 "views": 111,464 "category_id": 20,465 "featured_link": null,466 "has_accepted_answer": false,467 "posters": [468 {469 "extras": "latest single",470 "description": "Original Poster, Most Recent Poster",471 "user": {472 "id": 80911,473 "username": "mingyu",474 "name": "mingyu",475 "avatar_template": "/user_avatar/discuss.pytorch.org/mingyu/{size}/73994_2.png",476 "trust_level": 0477 }478 }479 ]480 },481 {482 "fancy_title": "Sharing CUDA tensor between different processes and pytorch versions",483 "id": 215253,484 "title": "Sharing CUDA tensor between different processes and pytorch versions",485 "slug": "sharing-cuda-tensor-between-different-processes-and-pytorch-versions",486 "posts_count": 1,487 "reply_count": 0,488 "highest_post_number": 1,489 "image_url": null,490 "created_at": "2025-01-11T10:16:36.400Z",491 "last_posted_at": "2025-01-11T10:16:36.488Z",492 "bumped": true,493 "bumped_at": "2025-01-11T11:36:27.908Z",494 "archetype": "regular",495 "unseen": false,496 "pinned": false,497 "unpinned": null,498 "visible": true,499 "closed": false,500 "archived": false,501 "bookmarked": null,502 "liked": null,503 "tags_descriptions": {},504 "like_count": 0,505 "views": 500,506 "category_id": 20,507 "featured_link": null,508 "has_accepted_answer": false,509 "posters": [510 {511 "extras": "latest single",512 "description": "Original Poster, Most Recent Poster",513 "user": {514 "id": 28905,515 "username": "braindotai",516 "name": "",517 "avatar_template": "/user_avatar/discuss.pytorch.org/braindotai/{size}/61373_2.png",518 "trust_level": 2519 }520 }521 ]522 },523 {524 "fancy_title": "Windows DDP on RTX 50-series only: use_libuv was requested but PyTorch was built without libuv support (works on 40/20-series)",525 "id": 223698,526 "title": "Windows DDP on RTX 50-series only: use_libuv was requested but PyTorch was built without libuv support (works on 40/20-series)",527 "slug": "windows-ddp-on-rtx-50-series-only-use-libuv-was-requested-but-pytorch-was-built-without-libuv-support-works-on-40-20-series",528 "posts_count": 1,529 "reply_count": 0,530 "highest_post_number": 1,531 "image_url": "https://discuss.pytorch.org/uploads/default/optimized/3X/3/f/3f05b13d0ace9f4ee92a2c3fb318271883fc2fb4_2_1024x532.jpeg",532 "created_at": "2025-10-25T03:07:02.896Z",533 "last_posted_at": "2025-10-25T03:07:02.965Z",534 "bumped": true,535 "bumped_at": "2025-10-25T03:07:02.965Z",536 "archetype": "regular",537 "unseen": false,538 "pinned": false,539 "unpinned": null,540 "visible": true,541 "closed": false,542 "archived": false,543 "bookmarked": null,544 "liked": null,545 "tags_descriptions": {},546 "like_count": 0,547 "views": 11,548 "category_id": 20,549 "featured_link": null,550 "has_accepted_answer": false,551 "posters": [552 {553 "extras": "latest single",554 "description": "Original Poster, Most Recent Poster",555 "user": {556 "id": 86279,557 "username": "Swati_sd",558 "name": "Swati Sanghamitra Das",559 "avatar_template": "/user_avatar/discuss.pytorch.org/swati_sd/{size}/78503_2.png",560 "trust_level": 0561 }562 }563 ]564 },565 {566 "fancy_title": "Implementation of Hierarchical Actor Critic with PPolicy-on Policy-off Policy Optimization for primitive actions",567 "id": 223612,568 "title": "Implementation of Hierarchical Actor Critic with PPolicy-on Policy-off Policy Optimization for primitive actions",569 "slug": "implementation-of-hierarchical-actor-critic-with-ppolicy-on-policy-off-policy-optimization-for-primitive-actions",570 "posts_count": 1,571 "reply_count": 0,572 "highest_post_number": 1,573 "image_url": null,574 "created_at": "2025-10-15T08:51:34.979Z",575 "last_posted_at": "2025-10-15T08:51:35.041Z",576 "bumped": true,577 "bumped_at": "2025-10-15T08:51:35.041Z",578 "archetype": "regular",579 "unseen": false,580 "pinned": false,581 "unpinned": null,582 "visible": true,583 "closed": false,584 "archived": false,585 "bookmarked": null,586 "liked": null,587 "tags_descriptions": {},588 "like_count": 0,589 "views": 12,590 "category_id": 6,591 "featured_link": null,592 "has_accepted_answer": false,593 "posters": [594 {595 "extras": "latest single",596 "description": "Original Poster, Most Recent Poster",597 "user": {598 "id": 80968,599 "username": "Lordking1624",600 "name": "",601 "avatar_template": "/letter_avatar_proxy/v4/letter/l/b38774/{size}.png",602 "trust_level": 1603 }604 }605 ]606 }607 ],608 "tags_descriptions": {},609 "fancy_title": "Having problem in using DistributedDataParallel.The script is just waiting for other clients",610 "id": 106281,611 "title": "Having problem in using DistributedDataParallel.The script is just waiting for other clients",612 "posts_count": 6,613 "created_at": "2020-12-16T06:56:37.099Z",614 "views": 1114,615 "reply_count": 3,616 "like_count": 0,617 "last_posted_at": "2020-12-17T04:43:06.552Z",618 "visible": true,619 "closed": false,620 "archived": false,621 "has_summary": false,622 "archetype": "regular",623 "slug": "having-problem-in-using-distributeddataparallel-the-script-is-just-waiting-for-other-clients",624 "category_id": 20,625 "word_count": 1652,626 "deleted_at": null,627 "user_id": 26932,628 "featured_link": null,629 "pinned_globally": false,630 "pinned_at": null,631 "pinned_until": null,632 "image_url": null,633 "slow_mode_seconds": 0,634 "draft": null,635 "draft_key": "topic_106281",636 "draft_sequence": null,637 "unpinned": null,638 "pinned": false,639 "current_post_number": 1,640 "highest_post_number": 6,641 "deleted_by": null,642 "actions_summary": [643 {644 "id": 4,645 "count": 0,646 "hidden": false,647 "can_act": false648 },649 {650 "id": 8,651 "count": 0,652 "hidden": false,653 "can_act": false654 },655 {656 "id": 10,657 "count": 0,658 "hidden": false,659 "can_act": false660 },661 {662 "id": 7,663 "count": 0,664 "hidden": false,665 "can_act": false666 }667 ],668 "chunk_size": 20,669 "bookmarked": false,670 "topic_timer": null,671 "message_bus_last_id": 0,672 "participant_count": 3,673 "show_read_indicator": false,674 "thumbnails": null,675 "slow_mode_enabled_until": null,676 "can_vote": false,677 "vote_count": 0,678 "user_voted": false,679 "discourse_zendesk_plugin_zendesk_id": null,680 "discourse_zendesk_plugin_zendesk_url": "https://your-url.zendesk.com/agent/tickets/",681 "details": {682 "can_edit": false,683 "notification_level": 1,684 "participants": [685 {686 "id": 26932,687 "username": "Sanjayvarma11",688 "name": "Gadiraju sanjay varma",689 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png",690 "post_count": 4,691 "primary_group_name": null,692 "flair_name": null,693 "flair_url": null,694 "flair_color": null,695 "flair_bg_color": null,696 "flair_group_id": null,697 "trust_level": 2698 },699 {700 "id": 3534,701 "username": "ptrblck",702 "name": "",703 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",704 "post_count": 1,705 "primary_group_name": null,706 "flair_name": null,707 "flair_url": null,708 "flair_color": null,709 "flair_bg_color": null,710 "flair_group_id": null,711 "admin": true,712 "moderator": true,713 "trust_level": 2714 },715 {716 "id": 22425,717 "username": "rvarm1",718 "name": "Rohan Varma",719 "avatar_template": "/user_avatar/discuss.pytorch.org/rvarm1/{size}/15821_2.png",720 "post_count": 1,721 "primary_group_name": null,722 "flair_name": null,723 "flair_url": null,724 "flair_color": null,725 "flair_bg_color": null,726 "flair_group_id": null,727 "trust_level": 2728 }729 ],730 "created_by": {731 "id": 26932,732 "username": "Sanjayvarma11",733 "name": "Gadiraju sanjay varma",734 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png"735 },736 "last_poster": {737 "id": 26932,738 "username": "Sanjayvarma11",739 "name": "Gadiraju sanjay varma",740 "avatar_template": "/user_avatar/discuss.pytorch.org/sanjayvarma11/{size}/20339_2.png"741 },742 "links": [743 {744 "url": "https://github.com/pytorch/pytorch/pull/47749",745 "title": null,746 "internal": false,747 "attachment": false,748 "reflection": false,749 "clicks": 6,750 "user_id": 22425,751 "domain": "github.com",752 "root_domain": "github.com"753 },754 {755 "url": "https://pytorch.org/docs/stable/distributed.html#tcp-initialization",756 "title": "Distributed communication package - torch.distributed — PyTorch 1.7.0 documentation",757 "internal": false,758 "attachment": false,759 "reflection": false,760 "clicks": 5,761 "user_id": 22425,762 "domain": "pytorch.org",763 "root_domain": "pytorch.org"764 }765 ]766 },767 "bookmarks": []768 },769 {770 "post_stream": {771 "posts": [772 {773 "id": 251710,774 "name": "Krzysztof Smyl",775 "username": "smyl",776 "avatar_template": "/user_avatar/discuss.pytorch.org/smyl/{size}/32365_2.png",777 "created_at": "2020-12-16T01:56:56.673Z",778 "cooked": "<p>Since torch==1.6.0 version using both Torch and PyCuda together causes unpredictable failures, even if they never share any memory.</p>\n<p>After PyTorch update, I encountered many CUDA errors in various places (illegal memory access, misaligned address, cuDNN error: CUDNN_STATUS_MAPPING_ERROR, etc.). I was finally able to pin it down to PyCuda usage, getting rid of all the PyCuda calls and imports fixes the problem. I prepared a simplified MNIST example (based on <a href=\"https://github.com/pytorch/examples/tree/master/mnist\" rel=\"noopener nofollow ugc\">https://github.com/pytorch/examples/tree/master/mnist</a>) that fails after unrelated time measurement using PyCuda events.</p>\n<p>Of course, this is far from a real use case, it just shows the kinds of operations that cause issues.</p>\n<p>I tested it with 1.4.0 (ok), 1.5.0 (ok), 1.6.0 (fails), 1.7.0 (fails), 1.7.1 (fails) and 1.8.0 nightly (fails).</p>\n<pre><code class=\"lang-auto\">import time\n\nimport pycuda.driver as cuda\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom pycuda.autoinit import context as pycuda_ctx\nfrom torchvision import datasets, transforms\n\n\nclass Net(nn.Module):\n def __init__(self):\n super(Net, self).__init__()\n self.conv1 = nn.Conv2d(1, 32, 3, 1)\n self.conv2 = nn.Conv2d(32, 64, 3, 1)\n self.dropout1 = nn.Dropout(0.25)\n self.dropout2 = nn.Dropout(0.5)\n self.fc1 = nn.Linear(9216, 128)\n self.fc2 = nn.Linear(128, 10)\n\n def forward(self, x):\n x = self.conv1(x)\n x = F.relu(x)\n x = self.conv2(x)\n x = F.relu(x)\n x = F.max_pool2d(x, 2)\n x = self.dropout1(x)\n x = torch.flatten(x, 1)\n x = self.fc1(x)\n x = F.relu(x)\n x = self.dropout2(x)\n x = self.fc2(x)\n output = F.log_softmax(x, dim=1)\n return output\n\n\ndef pycuda_dummy_measure_time():\n pycuda_ctx.push()\n event_start = cuda.Event().record()\n pycuda_ctx.pop()\n\n time.sleep(2)\n\n pycuda_ctx.push()\n event_stop = cuda.Event().record().synchronize()\n print(event_stop.time_since(event_start))\n pycuda_ctx.pop()\n\n\ndef main():\n torch.cuda.init() # any torch cuda initialization before pycuda calls, torch.randn(10).cuda() works too\n\n pycuda_dummy_measure_time() # measures time of a 2-second sleep using pycuda Events\n\n # normal MNIST training below\n transform = transforms.Compose([\n transforms.ToTensor(),\n transforms.Normalize((0.1307,), (0.3081,))\n ])\n dataset1 = datasets.MNIST('../data', train=True, download=True, transform=transform)\n train_loader = torch.utils.data.DataLoader(dataset1, batch_size=64, num_workers=1, pin_memory=True)\n\n model = Net().cuda()\n model.train()\n\n for batch_idx, (data, target) in enumerate(train_loader):\n data, target = data.cuda(), target.cuda()\n output = model(data)\n loss = F.nll_loss(output, target)\n loss.backward()\n if batch_idx % 10 == 0:\n print('[{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(batch_idx * len(data), len(train_loader.dataset),\n 100. * batch_idx / len(train_loader), loss.item()))\n\n\nif __name__ == '__main__':\n main()\n</code></pre>\n<p>On failing configuration it produces:</p>\n<pre><code class=\"lang-auto\">Traceback (most recent call last):\n File \"mnist.py\", line 77, in <module>\n main()\n File \"mnist.py\", line 68, in main\n output = model(data)\n File \"/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"mnist.py\", line 22, in forward\n x = self.conv1(x)\n File \"/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py\", line 423, in forward\n return self._conv_forward(input, self.weight)\n File \"/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py\", line 420, in _conv_forward\n self.padding, self.dilation, self.groups)\nRuntimeError: cuDNN error: CUDNN_STATUS_MAPPING_ERROR\n</code></pre>\n<p>All the configurations have:</p>\n<p>Ubuntu 18.04<br>\nPython 3.6.9<br>\npycuda==2020.1</p>\n<p>Some of the configurations I got the failures in:</p>\n<p>Tesla T4<br>\nDriver 440.64.00<br>\nCUDA 10.2, V10.2.89<br>\ntorch==1.6.0<br>\ntorchvision==0.7.0</p>\n<p>GeForce GTX 1070<br>\nDriver 450.80.02<br>\nCUDA 11.0, V11.0.221<br>\ntorch==1.7.1+cu110<br>\ntorchvision==0.8.2+cu110</p>\n<p>Some of the configurations with no problems:</p>\n<p>Tesla T4<br>\nDriver 440.64.00<br>\nCUDA 10.2, V10.2.89<br>\ntorch==1.4.0<br>\ntorchvision==0.5.0</p>\n<p>GeForce GTX 1070<br>\nDriver 450.80.02<br>\nCUDA 11.0, V11.0.221<br>\ntorch==1.5.0<br>\ntorchvision==0.6.0</p>\n<p>I was also doing some tests with different drivers and CUDA versions, but I didn’t write down the full configurations, so I don’t want to cause confusion. If I should check some configuration, please let me know.</p>\n<p>Thanks for all your help!</p>",779 "post_number": 1,780 "post_type": 1,781 "posts_count": 4,782 "updated_at": "2020-12-16T01:57:47.672Z",783 "reply_count": 0,784 "reply_to_post_number": null,785 "quote_count": 0,786 "incoming_link_count": 730,787 "reads": 23,788 "readers_count": 22,789 "score": 3654.6,790 "yours": false,791 "topic_id": 106261,792 "topic_slug": "pytorch-1-6-0-cannot-coexist-with-pycuda",793 "display_username": "Krzysztof Smyl",794 "primary_group_name": null,795 "flair_name": null,796 "flair_url": null,797 "flair_bg_color": null,798 "flair_color": null,799 "flair_group_id": null,800 "badges_granted": [],801 "version": 1,802 "can_edit": false,803 "can_delete": false,804 "can_recover": false,805 "can_see_hidden_post": false,806 "can_wiki": false,807 "link_counts": [808 {809 "url": "https://github.com/pytorch/examples/tree/master/mnist",810 "internal": false,811 "reflection": false,812 "title": "examples/mnist at master · pytorch/examples · GitHub",813 "clicks": 1814 }815 ],816 "read": true,817 "user_title": null,818 "bookmarked": false,819 "actions_summary": [],820 "moderator": false,821 "admin": false,822 "staff": false,823 "user_id": 40095,824 "hidden": false,825 "trust_level": 1,826 "deleted_at": null,827 "user_deleted": false,828 "edit_reason": null,829 "can_view_edit_history": true,830 "wiki": false,831 "post_url": "/t/pytorch-1-6-0-cannot-coexist-with-pycuda/106261/1",832 "can_accept_answer": false,833 "can_unaccept_answer": false,834 "accepted_answer": false,835 "topic_accepted_answer": null,836 "can_vote": false837 },838 {839 "id": 251762,840 "name": "",841 "username": "ptrblck",842 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",843 "created_at": "2020-12-16T08:00:16.502Z",844 "cooked": "<p>Are you using the same CUDA versions in PyTorch and PyCUDA?</p>",845 "post_number": 2,846 "post_type": 1,847 "posts_count": 4,848 "updated_at": "2020-12-16T08:00:16.502Z",849 "reply_count": 1,850 "reply_to_post_number": null,851 "quote_count": 0,852 "incoming_link_count": 115,853 "reads": 19,854 "readers_count": 18,855 "score": 583.8,856 "yours": false,857 "topic_id": 106261,858 "topic_slug": "pytorch-1-6-0-cannot-coexist-with-pycuda",859 "display_username": "",860 "primary_group_name": null,861 "flair_name": null,862 "flair_url": null,863 "flair_bg_color": null,864 "flair_color": null,865 "flair_group_id": null,866 "badges_granted": [],867 "version": 1,868 "can_edit": false,869 "can_delete": false,870 "can_recover": false,871 "can_see_hidden_post": false,872 "can_wiki": false,873 "read": true,874 "user_title": "",875 "bookmarked": false,876 "actions_summary": [],877 "moderator": true,878 "admin": true,879 "staff": true,880 "user_id": 3534,881 "hidden": false,882 "trust_level": 2,883 "deleted_at": null,884 "user_deleted": false,885 "edit_reason": null,886 "can_view_edit_history": true,887 "wiki": false,888 "post_url": "/t/pytorch-1-6-0-cannot-coexist-with-pycuda/106261/2",889 "can_accept_answer": false,890 "can_unaccept_answer": false,891 "accepted_answer": false,892 "topic_accepted_answer": null893 },894 {895 "id": 251846,896 "name": "Krzysztof Smyl",897 "username": "smyl",898 "avatar_template": "/user_avatar/discuss.pytorch.org/smyl/{size}/32365_2.png",899 "created_at": "2020-12-16T12:40:20.323Z",900 "cooked": "<p>Yes, they both use the default CUDA installation (/usr/local/cuda).</p>\n<p>I found a solution that works for me, I’m posting it here for any future readers.</p>\n<p>It turns out that since PyCuda 2020.1 version (released in October 2020) it is no longer required to create the PyCuda context, <a href=\"https://documen.tician.de/pycuda/driver.html#pycuda.driver.Device.retain_primary_context\" rel=\"noopener nofollow ugc\">retain_primary_context</a> method was added - it returns the device’s primary context. Using <code>retain_primary_context</code> instead of <code>import pycuda.autoinit</code> or <code>make_default_context</code> prevents new context creation and all the problems related to it.</p>\n<p>It still doesn’t explain why pycuda context coexistence with torch worked until torch 1.5.0 and stopped working afterwards, but I think that won’t matter anyway in most cases, <code>retain_primary_context</code> is cleaner than creating a new one.</p>\n<p>An improved version of the snippet above that works both for older and newer torch versions with pycuda>=2020.1:</p>\n<pre><code class=\"lang-auto\">import time\n\nimport pycuda.driver as cuda\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\n\ncuda.init()\npycuda_ctx = cuda.Device(0).retain_primary_context()\n\n\nclass Net(nn.Module):\n def __init__(self):\n super(Net, self).__init__()\n self.conv1 = nn.Conv2d(1, 32, 3, 1)\n self.conv2 = nn.Conv2d(32, 64, 3, 1)\n self.dropout1 = nn.Dropout(0.25)\n self.dropout2 = nn.Dropout(0.5)\n self.fc1 = nn.Linear(9216, 128)\n self.fc2 = nn.Linear(128, 10)\n\n def forward(self, x):\n x = self.conv1(x)\n x = F.relu(x)\n x = self.conv2(x)\n x = F.relu(x)\n x = F.max_pool2d(x, 2)\n x = self.dropout1(x)\n x = torch.flatten(x, 1)\n x = self.fc1(x)\n x = F.relu(x)\n x = self.dropout2(x)\n x = self.fc2(x)\n output = F.log_softmax(x, dim=1)\n return output\n\n\ndef pycuda_dummy_measure_time():\n pycuda_ctx.push()\n event_start = cuda.Event().record()\n pycuda_ctx.pop()\n\n time.sleep(2)\n\n pycuda_ctx.push()\n event_stop = cuda.Event().record().synchronize()\n print(event_stop.time_since(event_start))\n pycuda_ctx.pop()\n\n\ndef main():\n torch.cuda.init() # any torch cuda initialization before pycuda calls, torch.randn(10).cuda() works too\n\n pycuda_dummy_measure_time() # measures time of a 2-second sleep using pycuda Events\n\n # normal MNIST training below\n transform = transforms.Compose([\n transforms.ToTensor(),\n transforms.Normalize((0.1307,), (0.3081,))\n ])\n dataset1 = datasets.MNIST('../data', train=True, download=True, transform=transform)\n train_loader = torch.utils.data.DataLoader(dataset1, batch_size=64, num_workers=1, pin_memory=True)\n\n model = Net().cuda()\n model.train()\n\n for batch_idx, (data, target) in enumerate(train_loader):\n data, target = data.cuda(), target.cuda()\n output = model(data)\n loss = F.nll_loss(output, target)\n loss.backward()\n if batch_idx % 10 == 0:\n print('[{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(batch_idx * len(data), len(train_loader.dataset),\n 100. * batch_idx / len(train_loader), loss.item()))\n\n\nif __name__ == '__main__':\n main()\n</code></pre>",901 "post_number": 3,902 "post_type": 1,903 "posts_count": 4,904 "updated_at": "2020-12-16T12:40:20.323Z",905 "reply_count": 1,906 "reply_to_post_number": 2,907 "quote_count": 0,908 "incoming_link_count": 27,909 "reads": 19,910 "readers_count": 18,911 "score": 203.8,912 "yours": false,913 "topic_id": 106261,914 "topic_slug": "pytorch-1-6-0-cannot-coexist-with-pycuda",915 "display_username": "Krzysztof Smyl",916 "primary_group_name": null,917 "flair_name": null,918 "flair_url": null,919 "flair_bg_color": null,920 "flair_color": null,921 "flair_group_id": null,922 "badges_granted": [],923 "version": 1,924 "can_edit": false,925 "can_delete": false,926 "can_recover": false,927 "can_see_hidden_post": false,928 "can_wiki": false,929 "link_counts": [930 {931 "url": "https://documen.tician.de/pycuda/driver.html#pycuda.driver.Device.retain_primary_context",932 "internal": false,933 "reflection": false,934 "title": "Device Interface - PyCUDA 2020.1 documentation",935 "clicks": 28936 }937 ],938 "read": true,939 "user_title": null,940 "reply_to_user": {941 "id": 3534,942 "username": "ptrblck",943 "name": "",944 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png"945 },946 "bookmarked": false,947 "actions_summary": [948 {949 "id": 2,950 "count": 2951 }952 ],953 "moderator": false,954 "admin": false,955 "staff": false,956 "user_id": 40095,957 "hidden": false,958 "trust_level": 1,959 "deleted_at": null,960 "user_deleted": false,961 "edit_reason": null,962 "can_view_edit_history": true,963 "wiki": false,964 "post_url": "/t/pytorch-1-6-0-cannot-coexist-with-pycuda/106261/3",965 "can_accept_answer": false,966 "can_unaccept_answer": false,967 "accepted_answer": false,968 "topic_accepted_answer": null969 },970 {971 "id": 252013,972 "name": "",973 "username": "ptrblck",974 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",975 "created_at": "2020-12-17T03:46:29.215Z",976 "cooked": "<p>Thanks for the update! <img src=\"https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=9\" title=\":slight_smile:\" class=\"emoji\" alt=\":slight_smile:\"></p>",977 "post_number": 4,978 "post_type": 1,979 "posts_count": 4,980 "updated_at": "2020-12-17T03:46:29.215Z",981 "reply_count": 0,982 "reply_to_post_number": 3,983 "quote_count": 0,984 "incoming_link_count": 4,985 "reads": 17,986 "readers_count": 16,987 "score": 23.4,988 "yours": false,989 "topic_id": 106261,990 "topic_slug": "pytorch-1-6-0-cannot-coexist-with-pycuda",991 "display_username": "",992 "primary_group_name": null,993 "flair_name": null,994 "flair_url": null,995 "flair_bg_color": null,996 "flair_color": null,997 "flair_group_id": null,998 "badges_granted": [],999 "version": 1,1000 "can_edit": false,1001 "can_delete": false,1002 "can_recover": false,1003 "can_see_hidden_post": false,1004 "can_wiki": false,1005 "read": true,1006 "user_title": "",1007 "reply_to_user": {1008 "id": 40095,1009 "username": "smyl",1010 "name": "Krzysztof Smyl",1011 "avatar_template": "/user_avatar/discuss.pytorch.org/smyl/{size}/32365_2.png"1012 },1013 "bookmarked": false,1014 "actions_summary": [],1015 "moderator": true,1016 "admin": true,1017 "staff": true,1018 "user_id": 3534,1019 "hidden": false,1020 "trust_level": 2,1021 "deleted_at": null,1022 "user_deleted": false,1023 "edit_reason": null,1024 "can_view_edit_history": true,1025 "wiki": false,1026 "post_url": "/t/pytorch-1-6-0-cannot-coexist-with-pycuda/106261/4",1027 "can_accept_answer": false,1028 "can_unaccept_answer": false,1029 "accepted_answer": false,1030 "topic_accepted_answer": null1031 }1032 ],1033 "stream": [1034 251710,1035 251762,1036 251846,1037 2520131038 ]1039 },1040 "timeline_lookup": [1041 [1042 1,1043 17751044 ],1045 [1046 3,1047 17741048 ]1049 ],1050 "suggested_topics": [1051 {1052 "fancy_title": "Pytorch Weighted Regression Loss Functions",1053 "id": 212480,1054 "title": "Pytorch Weighted Regression Loss Functions",1055 "slug": "pytorch-weighted-regression-loss-functions",1056 "posts_count": 4,1057 "reply_count": 2,1058 "highest_post_number": 4,1059 "image_url": null,1060 "created_at": "2024-11-03T17:06:32.230Z",1061 "last_posted_at": "2024-11-04T23:07:20.590Z",1062 "bumped": true,1063 "bumped_at": "2024-11-04T23:07:20.590Z",1064 "archetype": "regular",1065 "unseen": false,1066 "pinned": false,1067 "unpinned": null,1068 "visible": true,1069 "closed": false,1070 "archived": false,1071 "bookmarked": null,1072 "liked": null,1073 "tags_descriptions": {},1074 "like_count": 1,1075 "views": 530,1076 "category_id": 1,1077 "featured_link": null,1078 "has_accepted_answer": true,1079 "posters": [1080 {1081 "extras": null,1082 "description": "Original Poster",1083 "user": {1084 "id": 80345,1085 "username": "Ajaikrish",1086 "name": "Ajaikrishna Ramalingam",1087 "avatar_template": "/user_avatar/discuss.pytorch.org/ajaikrish/{size}/72800_2.png",1088 "trust_level": 11089 }1090 },1091 {1092 "extras": "latest",1093 "description": "Most Recent Poster, Accepted Answer",1094 "user": {1095 "id": 3534,1096 "username": "ptrblck",1097 "name": "",1098 "avatar_template": "/user_avatar/discuss.pytorch.org/ptrblck/{size}/1823_2.png",1099 "admin": true,1100 "moderator": true,1101 "trust_level": 21102 }1103 }1104 ]1105 },1106 {1107 "fancy_title": "Unable to Import PyTorch After Upgrade in Docker Environment",1108 "id": 216366,1109 "title": "Unable to Import PyTorch After Upgrade in Docker Environment",1110 "slug": "unable-to-import-pytorch-after-upgrade-in-docker-environment",1111 "posts_count": 2,1112 "reply_count": 0,1113 "highest_post_number": 2,1114 "image_url": null,1115 "created_at": "2025-02-07T15:43:40.805Z",1116 "last_posted_at": "2025-02-12T22:21:57.537Z",1117 "bumped": true,1118 "bumped_at": "2025-02-12T22:21:57.537Z",1119 "archetype": "regular",1120 "unseen": false,1121 "pinned": false,1122 "unpinned": null,1123 "visible": true,1124 "closed": false,1125 "archived": false,1126 "bookmarked": null,1127 "liked": null,1128 "tags_descriptions": {},1129 "like_count": 0,1130 "views": 123,1131 "category_id": 1,1132 "featured_link": "https://github.com/pytorch/pytorch/issues/146701",1133 "featured_link_root_domain": "github.com",1134 "has_accepted_answer": false,1135 "posters": [1136 {1137 "extras": null,1138 "description": "Original Poster",1139 "user": {1140 "id": 25295,1141 "username": "unbreading",1142 "name": "",1143 "avatar_template": "/user_avatar/discuss.pytorch.org/unbreading/{size}/18533_2.png",1144 "trust_level": 11145 }1146 },1147 {1148 "extras": "latest",1149 "description": "Most Recent Poster",1150 "user": {1151 "id": 82649,1152 "username": "onedeadmatch",1153 "name": "dani",1154 "avatar_template": "/user_avatar/discuss.pytorch.org/onedeadmatch/{size}/75627_2.png",1155 "trust_level": 11156 }1157 }1158 ]1159 },1160 {1161 "fancy_title": "Generating random tensor based on seed tensor",1162 "id": 213855,1163 "title": "Generating random tensor based on seed tensor",1164 "slug": "generating-random-tensor-based-on-seed-tensor",1165 "posts_count": 5,1166 "reply_count": 3,1167 "highest_post_number": 5,1168 "image_url": null,1169 "created_at": "2024-12-05T13:42:09.731Z",1170 "last_posted_at": "2024-12-05T20:16:40.984Z",1171 "bumped": true,1172 "bumped_at": "2024-12-05T20:16:40.984Z",1173 "archetype": "regular",1174 "unseen": false,1175 "pinned": false,1176 "unpinned": null,1177 "visible": true,1178 "closed": false,1179 "archived": false,1180 "bookmarked": null,1181 "liked": null,1182 "tags_descriptions": {},1183 "like_count": 0,1184 "views": 247,1185 "category_id": 1,1186 "featured_link": null,1187 "has_accepted_answer": false,1188 "posters": [1189 {1190 "extras": "latest",1191 "description": "Original Poster, Most Recent Poster",1192 "user": {1193 "id": 81100,1194 "username": "Space4444",1195 "name": "Space4444",1196 "avatar_template": "/user_avatar/discuss.pytorch.org/space4444/{size}/74167_2.png",1197 "trust_level": 11198 }1199 },1200 {