sunilsarolkar/ISL-SignLanguageTranslation
2
1import torch2from collections import OrderedDict3 4import torch5import torch.nn as nn6 7# def make_layers(block, no_relu_layers):8# layers = []9# for layer_name, v in block.items():10# if 'pool' in layer_name:11# layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],12# padding=v[2])13# layers.append((layer_name, layer))14# else:15# conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],16# kernel_size=v[2], stride=v[3],17# padding=v[4])18# layers.append((layer_name, conv2d))19# if layer_name not in no_relu_layers:20# layers.append(('relu_'+layer_name, nn.ReLU(inplace=True)))21 22# return nn.Sequential(OrderedDict(layers))23 24 25def make_layers(block, no_relu_layers,prelu_layers = []):26 layers = []27 28 for layer_name, v in block.items():29 if 'pool' in layer_name:30 layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],31 padding=v[2])32 layers.append((layer_name, layer))33 else:34 #[3, 64, 3, 1, 1]35 conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],36 kernel_size=v[2], stride=v[3],37 padding=v[4])38 layers.append((layer_name, conv2d))39 if layer_name not in no_relu_layers:40 if layer_name not in prelu_layers:41 layers.append(('relu_'+layer_name, nn.ReLU(inplace=True)))42 else:43 layers.append(('prelu'+layer_name[4:],nn.PReLU(v[1])))44 45 return nn.Sequential(OrderedDict(layers))46 47 48def make_layers_Mconv(block,no_relu_layers):49 modules = []50 for layer_name, v in block.items():51 layers = []52 if 'pool' in layer_name:53 layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1],54 padding=v[2])55 layers.append((layer_name, layer))56 else:57 conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1],58 kernel_size=v[2], stride=v[3],59 padding=v[4])60 layers.append((layer_name, conv2d))61 if layer_name not in no_relu_layers:62 layers.append(('Mprelu'+layer_name[5:], nn.PReLU(v[1])))63 modules.append(nn.Sequential(OrderedDict(layers)))64 return nn.ModuleList(modules)65 66class bodypose_25_model(nn.Module):67 def __init__(self):68 super(bodypose_25_model,self).__init__()69 # these layers have no relu layer70 no_relu_layers = ['Mconv7_stage0_L1','Mconv7_stage0_L2',\71 'Mconv7_stage1_L1', 'Mconv7_stage1_L2',\72 'Mconv7_stage2_L2', 'Mconv7_stage3_L2']73 prelu_layers = ['conv4_2','conv4_3_CPM','conv4_4_CPM']74 blocks = {}75 block0 = OrderedDict([76 ('conv1_1', [3, 64, 3, 1, 1]),77 ('conv1_2', [64, 64, 3, 1, 1]),78 ('pool1_stage1', [2, 2, 0]),79 ('conv2_1', [64, 128, 3, 1, 1]),80 ('conv2_2', [128, 128, 3, 1, 1]),81 ('pool2_stage1', [2, 2, 0]),82 ('conv3_1', [128, 256, 3, 1, 1]),83 ('conv3_2', [256, 256, 3, 1, 1]),84 ('conv3_3', [256, 256, 3, 1, 1]),85 ('conv3_4', [256, 256, 3, 1, 1]),86 ('pool3_stage1', [2, 2, 0]),87 ('conv4_1', [256, 512, 3, 1, 1]),88 ('conv4_2', [512, 512, 3, 1, 1]),89 ('conv4_3_CPM', [512, 256, 3, 1, 1]),90 ('conv4_4_CPM', [256, 128, 3, 1, 1])91 ])92 self.model0 = make_layers(block0, no_relu_layers,prelu_layers)93 94 #L295 #stage096 blocks['Mconv1_stage0_L2'] = OrderedDict([97 ('Mconv1_stage0_L2_0',[128,96,3,1,1]),98 ('Mconv1_stage0_L2_1',[96,96,3,1,1]),99 ('Mconv1_stage0_L2_2',[96,96,3,1,1])100 ])101 for i in range(2,6):102 blocks['Mconv%d_stage0_L2' % i] = OrderedDict([103 ('Mconv%d_stage0_L2_0' % i,[288,96,3,1,1]),104 ('Mconv%d_stage0_L2_1' % i,[96,96,3,1,1]),105 ('Mconv%d_stage0_L2_2' % i,[96,96,3,1,1])106 ])107 blocks['Mconv6_7_stage0_L2'] = OrderedDict([108 ('Mconv6_stage0_L2',[288, 256, 1,1,0]),109 ('Mconv7_stage0_L2',[256,52,1,1,0])110 ])111 #stage1~3112 for s in range(1,4):113 blocks['Mconv1_stage%d_L2' % s] = OrderedDict([114 ('Mconv1_stage%d_L2_0' % s,[180,128,3,1,1]),115 ('Mconv1_stage%d_L2_1' % s,[128,128,3,1,1]),116 ('Mconv1_stage%d_L2_2' % s,[128,128,3,1,1])117 ])118 for i in range(2,6):119 blocks['Mconv%d_stage%d_L2' % (i,s)] = OrderedDict([120 ('Mconv%d_stage%d_L2_0' % (i,s) ,[384,128,3,1,1]),121 ('Mconv%d_stage%d_L2_1' % (i,s) ,[128,128,3,1,1]),122 ('Mconv%d_stage%d_L2_2' % (i,s) ,[128,128,3,1,1])123 ])124 blocks['Mconv6_7_stage%d_L2' % s] = OrderedDict([125 ('Mconv6_stage%d_L2' % s,[384,512,1,1,0]),126 ('Mconv7_stage%d_L2' % s,[512,52,1,1,0])127 ])128 129 #L1130 #stage0131 blocks['Mconv1_stage0_L1'] = OrderedDict([132 ('Mconv1_stage0_L1_0',[180,96,3,1,1]),133 ('Mconv1_stage0_L1_1',[96,96,3,1,1]),134 ('Mconv1_stage0_L1_2',[96,96,3,1,1])135 ])136 for i in range(2,6):137 blocks['Mconv%d_stage0_L1' % i] = OrderedDict([138 ('Mconv%d_stage0_L1_0' % i,[288,96,3,1,1]),139 ('Mconv%d_stage0_L1_1' % i,[96,96,3,1,1]),140 ('Mconv%d_stage0_L1_2' % i,[96,96,3,1,1])141 ])142 blocks['Mconv6_7_stage0_L1'] = OrderedDict([143 ('Mconv6_stage0_L1',[288, 256, 1,1,0]),144 ('Mconv7_stage0_L1',[256,26,1,1,0])145 ])146 #stage1147 blocks['Mconv1_stage1_L1'] = OrderedDict([148 ('Mconv1_stage1_L1_0',[206,128,3,1,1]),149 ('Mconv1_stage1_L1_1',[128,128,3,1,1]),150 ('Mconv1_stage1_L1_2',[128,128,3,1,1])151 ])152 for i in range(2,6):153 blocks['Mconv%d_stage1_L1' % i] = OrderedDict([154 ('Mconv%d_stage1_L1_0' % i,[384,128,3,1,1]),155 ('Mconv%d_stage1_L1_1' % i,[128,128,3,1,1]),156 ('Mconv%d_stage1_L1_2' % i,[128,128,3,1,1])157 ])158 blocks['Mconv6_7_stage1_L1'] = OrderedDict([159 ('Mconv6_stage1_L1',[384,512,1,1,0]),160 ('Mconv7_stage1_L1',[512,26,1,1,0])161 ])162 163 for k in blocks.keys():164 blocks[k] = make_layers_Mconv(blocks[k], no_relu_layers)165 self.models = nn.ModuleDict(blocks)166 #self.model_L2_S0_mconv1 = blocks['Mconv1_stage0_L2']167 for param in self.parameters():168 param.requires_grad = False169 170 171 def _Mconv_forward(self,x,models):172 outs = []173 out = x174 for m in models:175 out = m(out)176 outs.append(out)177 return torch.cat(outs,1)178 179 def forward(self,x):180 out0 = self.model0(x)181 #L2182 tout = out0183 for s in range(4):184 tout = self._Mconv_forward(tout,self.models['Mconv1_stage%d_L2' % s])185 for v in range(2,6):186 tout = self._Mconv_forward(tout,self.models['Mconv%d_stage%d_L2' % (v,s)])187 tout = self.models['Mconv6_7_stage%d_L2' % s][0](tout)188 tout = self.models['Mconv6_7_stage%d_L2' % s][1](tout)189 outL2 = tout190 tout = torch.cat([out0,tout],1)191 #L1 stage0192 #tout = torch.cat([out0,outL2],1)193 tout = self._Mconv_forward(tout, self.models['Mconv1_stage0_L1'])194 for v in range(2,6):195 tout = self._Mconv_forward(tout, self.models['Mconv%d_stage0_L1' % v])196 tout = self.models['Mconv6_7_stage0_L1'][0](tout)197 tout = self.models['Mconv6_7_stage0_L1'][1](tout)198 outS0L1 = tout199 tout = torch.cat([out0,outS0L1,outL2],1)200 #L1 stage1201 tout = self._Mconv_forward(tout, self.models['Mconv1_stage1_L1'])202 for v in range(2,6):203 tout = self._Mconv_forward(tout, self.models['Mconv%d_stage1_L1' % v])204 tout = self.models['Mconv6_7_stage1_L1'][0](tout)205 outS1L1 = self.models['Mconv6_7_stage1_L1'][1](tout)206 207 return outL2, outS1L1208 209 210class bodypose_model(nn.Module):211 def __init__(self):212 super(bodypose_model, self).__init__()213 214 # these layers have no relu layer215 no_relu_layers = ['conv5_5_CPM_L1', 'conv5_5_CPM_L2', 'Mconv7_stage2_L1',\216 'Mconv7_stage2_L2', 'Mconv7_stage3_L1', 'Mconv7_stage3_L2',\217 'Mconv7_stage4_L1', 'Mconv7_stage4_L2', 'Mconv7_stage5_L1',\218 'Mconv7_stage5_L2', 'Mconv7_stage6_L1', 'Mconv7_stage6_L1']219 blocks = {}220 block0 = OrderedDict([221 ('conv1_1', [3, 64, 3, 1, 1]),222 ('conv1_2', [64, 64, 3, 1, 1]),223 ('pool1_stage1', [2, 2, 0]),224 ('conv2_1', [64, 128, 3, 1, 1]),225 ('conv2_2', [128, 128, 3, 1, 1]),226 ('pool2_stage1', [2, 2, 0]),227 ('conv3_1', [128, 256, 3, 1, 1]),228 ('conv3_2', [256, 256, 3, 1, 1]),229 ('conv3_3', [256, 256, 3, 1, 1]),230 ('conv3_4', [256, 256, 3, 1, 1]),231 ('pool3_stage1', [2, 2, 0]),232 ('conv4_1', [256, 512, 3, 1, 1]),233 ('conv4_2', [512, 512, 3, 1, 1]),234 ('conv4_3_CPM', [512, 256, 3, 1, 1]),235 ('conv4_4_CPM', [256, 128, 3, 1, 1])236 ])237 238 239 # Stage 1240 block1_1 = OrderedDict([241 ('conv5_1_CPM_L1', [128, 128, 3, 1, 1]),242 ('conv5_2_CPM_L1', [128, 128, 3, 1, 1]),243 ('conv5_3_CPM_L1', [128, 128, 3, 1, 1]),244 ('conv5_4_CPM_L1', [128, 512, 1, 1, 0]),245 ('conv5_5_CPM_L1', [512, 38, 1, 1, 0])246 ])247 248 block1_2 = OrderedDict([249 ('conv5_1_CPM_L2', [128, 128, 3, 1, 1]),250 ('conv5_2_CPM_L2', [128, 128, 3, 1, 1]),251 ('conv5_3_CPM_L2', [128, 128, 3, 1, 1]),252 ('conv5_4_CPM_L2', [128, 512, 1, 1, 0]),253 ('conv5_5_CPM_L2', [512, 19, 1, 1, 0])254 ])255 blocks['block1_1'] = block1_1256 blocks['block1_2'] = block1_2257 258 self.model0 = make_layers(block0, no_relu_layers)259 260 # Stages 2 - 6261 for i in range(2, 7):262 blocks['block%d_1' % i] = OrderedDict([263 ('Mconv1_stage%d_L1' % i, [185, 128, 7, 1, 3]),264 ('Mconv2_stage%d_L1' % i, [128, 128, 7, 1, 3]),265 ('Mconv3_stage%d_L1' % i, [128, 128, 7, 1, 3]),266 ('Mconv4_stage%d_L1' % i, [128, 128, 7, 1, 3]),267 ('Mconv5_stage%d_L1' % i, [128, 128, 7, 1, 3]),268 ('Mconv6_stage%d_L1' % i, [128, 128, 1, 1, 0]),269 ('Mconv7_stage%d_L1' % i, [128, 38, 1, 1, 0])270 ])271 272 blocks['block%d_2' % i] = OrderedDict([273 ('Mconv1_stage%d_L2' % i, [185, 128, 7, 1, 3]),274 ('Mconv2_stage%d_L2' % i, [128, 128, 7, 1, 3]),275 ('Mconv3_stage%d_L2' % i, [128, 128, 7, 1, 3]),276 ('Mconv4_stage%d_L2' % i, [128, 128, 7, 1, 3]),277 ('Mconv5_stage%d_L2' % i, [128, 128, 7, 1, 3]),278 ('Mconv6_stage%d_L2' % i, [128, 128, 1, 1, 0]),279 ('Mconv7_stage%d_L2' % i, [128, 19, 1, 1, 0])280 ])281 282 for k in blocks.keys():283 blocks[k] = make_layers(blocks[k], no_relu_layers)284 285 self.model1_1 = blocks['block1_1']286 self.model2_1 = blocks['block2_1']287 self.model3_1 = blocks['block3_1']288 self.model4_1 = blocks['block4_1']289 self.model5_1 = blocks['block5_1']290 self.model6_1 = blocks['block6_1']291 292 self.model1_2 = blocks['block1_2']293 self.model2_2 = blocks['block2_2']294 self.model3_2 = blocks['block3_2']295 self.model4_2 = blocks['block4_2']296 self.model5_2 = blocks['block5_2']297 self.model6_2 = blocks['block6_2']298 for param in self.parameters():299 param.requires_grad = False300 301 302 def forward(self, x):303 304 out1 = self.model0(x)305 306 out1_1 = self.model1_1(out1)307 out1_2 = self.model1_2(out1)308 out2 = torch.cat([out1_1, out1_2, out1], 1)309 310 out2_1 = self.model2_1(out2)311 out2_2 = self.model2_2(out2)312 out3 = torch.cat([out2_1, out2_2, out1], 1)313 314 out3_1 = self.model3_1(out3)315 out3_2 = self.model3_2(out3)316 out4 = torch.cat([out3_1, out3_2, out1], 1)317 318 out4_1 = self.model4_1(out4)319 out4_2 = self.model4_2(out4)320 out5 = torch.cat([out4_1, out4_2, out1], 1)321 322 out5_1 = self.model5_1(out5)323 out5_2 = self.model5_2(out5)324 out6 = torch.cat([out5_1, out5_2, out1], 1)325 326 out6_1 = self.model6_1(out6)327 out6_2 = self.model6_2(out6)328 329 return out6_1, out6_2330 331class handpose_model(nn.Module):332 def __init__(self):333 super(handpose_model, self).__init__()334 335 # these layers have no relu layer336 no_relu_layers = ['conv6_2_CPM', 'Mconv7_stage2', 'Mconv7_stage3',\337 'Mconv7_stage4', 'Mconv7_stage5', 'Mconv7_stage6']338 # stage 1339 block1_0 = OrderedDict([340 ('conv1_1', [3, 64, 3, 1, 1]),341 ('conv1_2', [64, 64, 3, 1, 1]),342 ('pool1_stage1', [2, 2, 0]),343 ('conv2_1', [64, 128, 3, 1, 1]),344 ('conv2_2', [128, 128, 3, 1, 1]),345 ('pool2_stage1', [2, 2, 0]),346 ('conv3_1', [128, 256, 3, 1, 1]),347 ('conv3_2', [256, 256, 3, 1, 1]),348 ('conv3_3', [256, 256, 3, 1, 1]),349 ('conv3_4', [256, 256, 3, 1, 1]),350 ('pool3_stage1', [2, 2, 0]),351 ('conv4_1', [256, 512, 3, 1, 1]),352 ('conv4_2', [512, 512, 3, 1, 1]),353 ('conv4_3', [512, 512, 3, 1, 1]),354 ('conv4_4', [512, 512, 3, 1, 1]),355 ('conv5_1', [512, 512, 3, 1, 1]),356 ('conv5_2', [512, 512, 3, 1, 1]),357 ('conv5_3_CPM', [512, 128, 3, 1, 1])358 ])359 360 block1_1 = OrderedDict([361 ('conv6_1_CPM', [128, 512, 1, 1, 0]),362 ('conv6_2_CPM', [512, 22, 1, 1, 0])363 ])364 365 blocks = {}366 blocks['block1_0'] = block1_0367 blocks['block1_1'] = block1_1368 369 # stage 2-6370 for i in range(2, 7):371 blocks['block%d' % i] = OrderedDict([372 ('Mconv1_stage%d' % i, [150, 128, 7, 1, 3]),373 ('Mconv2_stage%d' % i, [128, 128, 7, 1, 3]),374 ('Mconv3_stage%d' % i, [128, 128, 7, 1, 3]),375 ('Mconv4_stage%d' % i, [128, 128, 7, 1, 3]),376 ('Mconv5_stage%d' % i, [128, 128, 7, 1, 3]),377 ('Mconv6_stage%d' % i, [128, 128, 1, 1, 0]),378 ('Mconv7_stage%d' % i, [128, 22, 1, 1, 0])379 ])380 381 for k in blocks.keys():382 blocks[k] = make_layers(blocks[k], no_relu_layers)383 384 self.model1_0 = blocks['block1_0']385 self.model1_1 = blocks['block1_1']386 self.model2 = blocks['block2']387 self.model3 = blocks['block3']388 self.model4 = blocks['block4']389 self.model5 = blocks['block5']390 self.model6 = blocks['block6']391 for param in self.parameters():392 param.requires_grad = False393 394 def forward(self, x):395 out1_0 = self.model1_0(x)396 out1_1 = self.model1_1(out1_0)397 concat_stage2 = torch.cat([out1_1, out1_0], 1)398 out_stage2 = self.model2(concat_stage2)399 concat_stage3 = torch.cat([out_stage2, out1_0], 1)400 out_stage3 = self.model3(concat_stage3)401 concat_stage4 = torch.cat([out_stage3, out1_0], 1)402 out_stage4 = self.model4(concat_stage4)403 concat_stage5 = torch.cat([out_stage4, out1_0], 1)404 out_stage5 = self.model5(concat_stage5)405 concat_stage6 = torch.cat([out_stage5, out1_0], 1)406 out_stage6 = self.model6(concat_stage6)407 return out_stage6