Krish778/AI_singer
0
1import copy2import math3import numpy as np4import scipy5import torch6from torch import nn7from torch.nn import functional as F8 9from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d10from torch.nn.utils import weight_norm, remove_weight_norm11 12from lib.infer_pack import commons13from lib.infer_pack.commons import init_weights, get_padding14from lib.infer_pack.transforms import piecewise_rational_quadratic_transform15 16 17LRELU_SLOPE = 0.118 19 20class LayerNorm(nn.Module):21 def __init__(self, channels, eps=1e-5):22 super().__init__()23 self.channels = channels24 self.eps = eps25 26 self.gamma = nn.Parameter(torch.ones(channels))27 self.beta = nn.Parameter(torch.zeros(channels))28 29 def forward(self, x):30 x = x.transpose(1, -1)31 x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)32 return x.transpose(1, -1)33 34 35class ConvReluNorm(nn.Module):36 def __init__(37 self,38 in_channels,39 hidden_channels,40 out_channels,41 kernel_size,42 n_layers,43 p_dropout,44 ):45 super().__init__()46 self.in_channels = in_channels47 self.hidden_channels = hidden_channels48 self.out_channels = out_channels49 self.kernel_size = kernel_size50 self.n_layers = n_layers51 self.p_dropout = p_dropout52 assert n_layers > 1, "Number of layers should be larger than 0."53 54 self.conv_layers = nn.ModuleList()55 self.norm_layers = nn.ModuleList()56 self.conv_layers.append(57 nn.Conv1d(58 in_channels, hidden_channels, kernel_size, padding=kernel_size // 259 )60 )61 self.norm_layers.append(LayerNorm(hidden_channels))62 self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))63 for _ in range(n_layers - 1):64 self.conv_layers.append(65 nn.Conv1d(66 hidden_channels,67 hidden_channels,68 kernel_size,69 padding=kernel_size // 2,70 )71 )72 self.norm_layers.append(LayerNorm(hidden_channels))73 self.proj = nn.Conv1d(hidden_channels, out_channels, 1)74 self.proj.weight.data.zero_()75 self.proj.bias.data.zero_()76 77 def forward(self, x, x_mask):78 x_org = x79 for i in range(self.n_layers):80 x = self.conv_layers[i](x * x_mask)81 x = self.norm_layers[i](x)82 x = self.relu_drop(x)83 x = x_org + self.proj(x)84 return x * x_mask85 86 87class DDSConv(nn.Module):88 """89 Dialted and Depth-Separable Convolution90 """91 92 def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):93 super().__init__()94 self.channels = channels95 self.kernel_size = kernel_size96 self.n_layers = n_layers97 self.p_dropout = p_dropout98 99 self.drop = nn.Dropout(p_dropout)100 self.convs_sep = nn.ModuleList()101 self.convs_1x1 = nn.ModuleList()102 self.norms_1 = nn.ModuleList()103 self.norms_2 = nn.ModuleList()104 for i in range(n_layers):105 dilation = kernel_size**i106 padding = (kernel_size * dilation - dilation) // 2107 self.convs_sep.append(108 nn.Conv1d(109 channels,110 channels,111 kernel_size,112 groups=channels,113 dilation=dilation,114 padding=padding,115 )116 )117 self.convs_1x1.append(nn.Conv1d(channels, channels, 1))118 self.norms_1.append(LayerNorm(channels))119 self.norms_2.append(LayerNorm(channels))120 121 def forward(self, x, x_mask, g=None):122 if g is not None:123 x = x + g124 for i in range(self.n_layers):125 y = self.convs_sep[i](x * x_mask)126 y = self.norms_1[i](y)127 y = F.gelu(y)128 y = self.convs_1x1[i](y)129 y = self.norms_2[i](y)130 y = F.gelu(y)131 y = self.drop(y)132 x = x + y133 return x * x_mask134 135 136class WN(torch.nn.Module):137 def __init__(138 self,139 hidden_channels,140 kernel_size,141 dilation_rate,142 n_layers,143 gin_channels=0,144 p_dropout=0,145 ):146 super(WN, self).__init__()147 assert kernel_size % 2 == 1148 self.hidden_channels = hidden_channels149 self.kernel_size = (kernel_size,)150 self.dilation_rate = dilation_rate151 self.n_layers = n_layers152 self.gin_channels = gin_channels153 self.p_dropout = p_dropout154 155 self.in_layers = torch.nn.ModuleList()156 self.res_skip_layers = torch.nn.ModuleList()157 self.drop = nn.Dropout(p_dropout)158 159 if gin_channels != 0:160 cond_layer = torch.nn.Conv1d(161 gin_channels, 2 * hidden_channels * n_layers, 1162 )163 self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight")164 165 for i in range(n_layers):166 dilation = dilation_rate**i167 padding = int((kernel_size * dilation - dilation) / 2)168 in_layer = torch.nn.Conv1d(169 hidden_channels,170 2 * hidden_channels,171 kernel_size,172 dilation=dilation,173 padding=padding,174 )175 in_layer = torch.nn.utils.weight_norm(in_layer, name="weight")176 self.in_layers.append(in_layer)177 178 # last one is not necessary179 if i < n_layers - 1:180 res_skip_channels = 2 * hidden_channels181 else:182 res_skip_channels = hidden_channels183 184 res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)185 res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight")186 self.res_skip_layers.append(res_skip_layer)187 188 def forward(self, x, x_mask, g=None, **kwargs):189 output = torch.zeros_like(x)190 n_channels_tensor = torch.IntTensor([self.hidden_channels])191 192 if g is not None:193 g = self.cond_layer(g)194 195 for i in range(self.n_layers):196 x_in = self.in_layers[i](x)197 if g is not None:198 cond_offset = i * 2 * self.hidden_channels199 g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :]200 else:201 g_l = torch.zeros_like(x_in)202 203 acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor)204 acts = self.drop(acts)205 206 res_skip_acts = self.res_skip_layers[i](acts)207 if i < self.n_layers - 1:208 res_acts = res_skip_acts[:, : self.hidden_channels, :]209 x = (x + res_acts) * x_mask210 output = output + res_skip_acts[:, self.hidden_channels :, :]211 else:212 output = output + res_skip_acts213 return output * x_mask214 215 def remove_weight_norm(self):216 if self.gin_channels != 0:217 torch.nn.utils.remove_weight_norm(self.cond_layer)218 for l in self.in_layers:219 torch.nn.utils.remove_weight_norm(l)220 for l in self.res_skip_layers:221 torch.nn.utils.remove_weight_norm(l)222 223 224class ResBlock1(torch.nn.Module):225 def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):226 super(ResBlock1, self).__init__()227 self.convs1 = nn.ModuleList(228 [229 weight_norm(230 Conv1d(231 channels,232 channels,233 kernel_size,234 1,235 dilation=dilation[0],236 padding=get_padding(kernel_size, dilation[0]),237 )238 ),239 weight_norm(240 Conv1d(241 channels,242 channels,243 kernel_size,244 1,245 dilation=dilation[1],246 padding=get_padding(kernel_size, dilation[1]),247 )248 ),249 weight_norm(250 Conv1d(251 channels,252 channels,253 kernel_size,254 1,255 dilation=dilation[2],256 padding=get_padding(kernel_size, dilation[2]),257 )258 ),259 ]260 )261 self.convs1.apply(init_weights)262 263 self.convs2 = nn.ModuleList(264 [265 weight_norm(266 Conv1d(267 channels,268 channels,269 kernel_size,270 1,271 dilation=1,272 padding=get_padding(kernel_size, 1),273 )274 ),275 weight_norm(276 Conv1d(277 channels,278 channels,279 kernel_size,280 1,281 dilation=1,282 padding=get_padding(kernel_size, 1),283 )284 ),285 weight_norm(286 Conv1d(287 channels,288 channels,289 kernel_size,290 1,291 dilation=1,292 padding=get_padding(kernel_size, 1),293 )294 ),295 ]296 )297 self.convs2.apply(init_weights)298 299 def forward(self, x, x_mask=None):300 for c1, c2 in zip(self.convs1, self.convs2):301 xt = F.leaky_relu(x, LRELU_SLOPE)302 if x_mask is not None:303 xt = xt * x_mask304 xt = c1(xt)305 xt = F.leaky_relu(xt, LRELU_SLOPE)306 if x_mask is not None:307 xt = xt * x_mask308 xt = c2(xt)309 x = xt + x310 if x_mask is not None:311 x = x * x_mask312 return x313 314 def remove_weight_norm(self):315 for l in self.convs1:316 remove_weight_norm(l)317 for l in self.convs2:318 remove_weight_norm(l)319 320 321class ResBlock2(torch.nn.Module):322 def __init__(self, channels, kernel_size=3, dilation=(1, 3)):323 super(ResBlock2, self).__init__()324 self.convs = nn.ModuleList(325 [326 weight_norm(327 Conv1d(328 channels,329 channels,330 kernel_size,331 1,332 dilation=dilation[0],333 padding=get_padding(kernel_size, dilation[0]),334 )335 ),336 weight_norm(337 Conv1d(338 channels,339 channels,340 kernel_size,341 1,342 dilation=dilation[1],343 padding=get_padding(kernel_size, dilation[1]),344 )345 ),346 ]347 )348 self.convs.apply(init_weights)349 350 def forward(self, x, x_mask=None):351 for c in self.convs:352 xt = F.leaky_relu(x, LRELU_SLOPE)353 if x_mask is not None:354 xt = xt * x_mask355 xt = c(xt)356 x = xt + x357 if x_mask is not None:358 x = x * x_mask359 return x360 361 def remove_weight_norm(self):362 for l in self.convs:363 remove_weight_norm(l)364 365 366class Log(nn.Module):367 def forward(self, x, x_mask, reverse=False, **kwargs):368 if not reverse:369 y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask370 logdet = torch.sum(-y, [1, 2])371 return y, logdet372 else:373 x = torch.exp(x) * x_mask374 return x375 376 377class Flip(nn.Module):378 def forward(self, x, *args, reverse=False, **kwargs):379 x = torch.flip(x, [1])380 if not reverse:381 logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)382 return x, logdet383 else:384 return x385 386 387class ElementwiseAffine(nn.Module):388 def __init__(self, channels):389 super().__init__()390 self.channels = channels391 self.m = nn.Parameter(torch.zeros(channels, 1))392 self.logs = nn.Parameter(torch.zeros(channels, 1))393 394 def forward(self, x, x_mask, reverse=False, **kwargs):395 if not reverse:396 y = self.m + torch.exp(self.logs) * x397 y = y * x_mask398 logdet = torch.sum(self.logs * x_mask, [1, 2])399 return y, logdet400 else:401 x = (x - self.m) * torch.exp(-self.logs) * x_mask402 return x403 404 405class ResidualCouplingLayer(nn.Module):406 def __init__(407 self,408 channels,409 hidden_channels,410 kernel_size,411 dilation_rate,412 n_layers,413 p_dropout=0,414 gin_channels=0,415 mean_only=False,416 ):417 assert channels % 2 == 0, "channels should be divisible by 2"418 super().__init__()419 self.channels = channels420 self.hidden_channels = hidden_channels421 self.kernel_size = kernel_size422 self.dilation_rate = dilation_rate423 self.n_layers = n_layers424 self.half_channels = channels // 2425 self.mean_only = mean_only426 427 self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)428 self.enc = WN(429 hidden_channels,430 kernel_size,431 dilation_rate,432 n_layers,433 p_dropout=p_dropout,434 gin_channels=gin_channels,435 )436 self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)437 self.post.weight.data.zero_()438 self.post.bias.data.zero_()439 440 def forward(self, x, x_mask, g=None, reverse=False):441 x0, x1 = torch.split(x, [self.half_channels] * 2, 1)442 h = self.pre(x0) * x_mask443 h = self.enc(h, x_mask, g=g)444 stats = self.post(h) * x_mask445 if not self.mean_only:446 m, logs = torch.split(stats, [self.half_channels] * 2, 1)447 else:448 m = stats449 logs = torch.zeros_like(m)450 451 if not reverse:452 x1 = m + x1 * torch.exp(logs) * x_mask453 x = torch.cat([x0, x1], 1)454 logdet = torch.sum(logs, [1, 2])455 return x, logdet456 else:457 x1 = (x1 - m) * torch.exp(-logs) * x_mask458 x = torch.cat([x0, x1], 1)459 return x460 461 def remove_weight_norm(self):462 self.enc.remove_weight_norm()463 464 465class ConvFlow(nn.Module):466 def __init__(467 self,468 in_channels,469 filter_channels,470 kernel_size,471 n_layers,472 num_bins=10,473 tail_bound=5.0,474 ):475 super().__init__()476 self.in_channels = in_channels477 self.filter_channels = filter_channels478 self.kernel_size = kernel_size479 self.n_layers = n_layers480 self.num_bins = num_bins481 self.tail_bound = tail_bound482 self.half_channels = in_channels // 2483 484 self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)485 self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0)486 self.proj = nn.Conv1d(487 filter_channels, self.half_channels * (num_bins * 3 - 1), 1488 )489 self.proj.weight.data.zero_()490 self.proj.bias.data.zero_()491 492 def forward(self, x, x_mask, g=None, reverse=False):493 x0, x1 = torch.split(x, [self.half_channels] * 2, 1)494 h = self.pre(x0)495 h = self.convs(h, x_mask, g=g)496 h = self.proj(h) * x_mask497 498 b, c, t = x0.shape499 h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]500 501 unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels)502 unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt(503 self.filter_channels504 )505 unnormalized_derivatives = h[..., 2 * self.num_bins :]506 507 x1, logabsdet = piecewise_rational_quadratic_transform(508 x1,509 unnormalized_widths,510 unnormalized_heights,511 unnormalized_derivatives,512 inverse=reverse,513 tails="linear",514 tail_bound=self.tail_bound,515 )516 517 x = torch.cat([x0, x1], 1) * x_mask518 logdet = torch.sum(logabsdet * x_mask, [1, 2])519 if not reverse:520 return x, logdet521 else:522 return x523