Aloento/9Nine-PITS
1
1# from https://github.com/jaywalnut310/vits2# from https://github.com/ncsoft/avocodo3import math4 5import torch6from torch import nn7from torch.nn import Conv1d, ConvTranspose1d, Conv2d8from torch.nn import functional as F9from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm10 11import attentions12import commons13import modules14from analysis import Pitch15from commons import init_weights, get_padding16from pqmf import PQMF17 18 19# for Q option20# from functions import vq, vq_st21 22 23class StochasticDurationPredictor(nn.Module):24 25 def __init__(self,26 in_channels,27 filter_channels,28 kernel_size,29 p_dropout,30 n_flows=4,31 gin_channels=0):32 super().__init__()33 # it needs to be removed from future version.34 filter_channels = in_channels35 self.in_channels = in_channels36 self.filter_channels = filter_channels37 self.kernel_size = kernel_size38 self.p_dropout = p_dropout39 self.n_flows = n_flows40 self.gin_channels = gin_channels41 42 self.log_flow = modules.Log()43 self.flows = nn.ModuleList()44 self.flows.append(modules.ElementwiseAffine(2))45 for i in range(n_flows):46 self.flows.append(47 modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))48 self.flows.append(modules.Flip())49 50 self.post_pre = nn.Conv1d(1, filter_channels, 1)51 self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)52 self.post_convs = modules.DDSConv(filter_channels,53 kernel_size,54 n_layers=3,55 p_dropout=p_dropout)56 self.post_flows = nn.ModuleList()57 self.post_flows.append(modules.ElementwiseAffine(2))58 for i in range(4):59 self.post_flows.append(60 modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))61 self.post_flows.append(modules.Flip())62 63 self.pre = nn.Conv1d(in_channels, filter_channels, 1)64 self.proj = nn.Conv1d(filter_channels, filter_channels, 1)65 self.convs = modules.DDSConv(filter_channels,66 kernel_size,67 n_layers=3,68 p_dropout=p_dropout)69 if gin_channels != 0:70 self.cond = nn.Conv1d(gin_channels, filter_channels, 1)71 72 def forward(self,73 x,74 x_mask,75 w=None,76 g=None,77 reverse=False,78 noise_scale=1.0):79 x = torch.detach(x)80 x = self.pre(x)81 if g is not None:82 g = torch.detach(g)83 x = x + self.cond(g)84 x = self.convs(x, x_mask)85 x = self.proj(x) * x_mask86 87 if not reverse:88 flows = self.flows89 assert w is not None90 91 logdet_tot_q = 092 h_w = self.post_pre(w)93 h_w = self.post_convs(h_w, x_mask)94 h_w = self.post_proj(h_w) * x_mask95 e_q = torch.randn(w.size(0), 2, w.size(2)).to(96 device=x.device, dtype=x.dtype) * x_mask97 z_q = e_q98 for flow in self.post_flows:99 z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))100 logdet_tot_q += logdet_q101 z_u, z1 = torch.split(z_q, [1, 1], 1)102 u = torch.sigmoid(z_u) * x_mask103 z0 = (w - u) * x_mask104 logdet_tot_q += torch.sum(105 (F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2])106 logq = torch.sum(107 -0.5 * (math.log(2 * math.pi) +108 (e_q ** 2)) * x_mask, [1, 2]) - logdet_tot_q109 110 logdet_tot = 0111 z0, logdet = self.log_flow(z0, x_mask)112 logdet_tot += logdet113 z = torch.cat([z0, z1], 1)114 for flow in flows:115 z, logdet = flow(z, x_mask, g=x, reverse=reverse)116 logdet_tot = logdet_tot + logdet117 nll = torch.sum(0.5 * (math.log(2 * math.pi) +118 (z ** 2)) * x_mask, [1, 2]) - logdet_tot119 return nll + logq # [b]120 else:121 flows = list(reversed(self.flows))122 flows = flows[:-2] + [flows[-1]] # remove a useless vflow123 z = torch.randn(x.size(0), 2, x.size(2)).to(124 device=x.device, dtype=x.dtype) * noise_scale125 for flow in flows:126 z = flow(z, x_mask, g=x, reverse=reverse)127 z0, z1 = torch.split(z, [1, 1], 1)128 logw = z0129 return logw130 131 132class DurationPredictor(nn.Module):133 134 def __init__(self,135 in_channels,136 filter_channels,137 kernel_size,138 p_dropout,139 gin_channels=0):140 super().__init__()141 142 self.in_channels = in_channels143 self.filter_channels = filter_channels144 self.kernel_size = kernel_size145 self.p_dropout = p_dropout146 self.gin_channels = gin_channels147 148 self.drop = nn.Dropout(p_dropout)149 self.conv_1 = nn.Conv1d(in_channels,150 filter_channels,151 kernel_size,152 padding=kernel_size // 2)153 self.norm_1 = modules.LayerNorm(filter_channels)154 self.conv_2 = nn.Conv1d(filter_channels,155 filter_channels,156 kernel_size,157 padding=kernel_size // 2)158 self.norm_2 = modules.LayerNorm(filter_channels)159 self.proj = nn.Conv1d(filter_channels, 1, 1)160 161 if gin_channels != 0:162 self.cond = nn.Conv1d(gin_channels, in_channels, 1)163 164 def forward(self, x, x_mask, g=None):165 x = torch.detach(x)166 if g is not None:167 g = torch.detach(g)168 x = x + self.cond(g)169 x = self.conv_1(x * x_mask)170 x = torch.relu(x)171 x = self.norm_1(x)172 x = self.drop(x)173 x = self.conv_2(x * x_mask)174 x = torch.relu(x)175 x = self.norm_2(x)176 x = self.drop(x)177 x = self.proj(x * x_mask)178 return x * x_mask179 180 181class TextEncoder(nn.Module):182 183 def __init__(self, n_vocab, out_channels, hidden_channels, filter_channels,184 n_heads, n_layers, kernel_size, p_dropout):185 super().__init__()186 self.n_vocab = n_vocab187 self.out_channels = out_channels188 self.hidden_channels = hidden_channels189 self.filter_channels = filter_channels190 self.n_heads = n_heads191 self.n_layers = n_layers192 self.kernel_size = kernel_size193 self.p_dropout = p_dropout194 195 self.emb = nn.Embedding(n_vocab, hidden_channels)196 nn.init.normal_(self.emb.weight, 0.0, hidden_channels ** -0.5)197 self.emb_t = nn.Embedding(6, hidden_channels)198 nn.init.normal_(self.emb_t.weight, 0.0, hidden_channels ** -0.5)199 200 self.encoder = attentions.Encoder(hidden_channels, filter_channels,201 n_heads, n_layers, kernel_size,202 p_dropout)203 self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)204 205 def forward(self, x, t, x_lengths):206 t_zero = (t == 0)207 emb_t = self.emb_t(t)208 emb_t[t_zero, :] = 0209 x = (self.emb(x) + emb_t) * math.sqrt(210 self.hidden_channels) # [b, t, h]211 # x = torch.transpose(x, 1, -1) # [b, h, t]212 x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(1)),213 1).to(x.dtype)214 # x = self.encoder(x * x_mask, x_mask)215 x = torch.einsum('btd,but->bdt', x, x_mask)216 x = self.encoder(x, x_mask)217 stats = self.proj(x) * x_mask218 219 m, logs = torch.split(stats, self.out_channels, dim=1)220 return x, m, logs, x_mask221 222 223class ResidualCouplingBlock(nn.Module):224 225 def __init__(self,226 channels,227 hidden_channels,228 kernel_size,229 dilation_rate,230 n_layers,231 n_flows=4,232 gin_channels=0):233 super().__init__()234 self.channels = channels235 self.hidden_channels = hidden_channels236 self.kernel_size = kernel_size237 self.dilation_rate = dilation_rate238 self.n_layers = n_layers239 self.n_flows = n_flows240 self.gin_channels = gin_channels241 242 self.flows = nn.ModuleList()243 for i in range(n_flows):244 self.flows.append(245 modules.ResidualCouplingLayer(channels,246 hidden_channels,247 kernel_size,248 dilation_rate,249 n_layers,250 gin_channels=gin_channels,251 mean_only=True))252 self.flows.append(modules.Flip())253 254 def forward(self, x, x_mask, g=None, reverse=False):255 if not reverse:256 for flow in self.flows:257 x, _ = flow(x, x_mask, g=g, reverse=reverse)258 else:259 for flow in reversed(self.flows):260 x = flow(x, x_mask, g=g, reverse=reverse)261 return x262 263 264class PosteriorEncoder(nn.Module):265 266 def __init__(self,267 in_channels,268 out_channels,269 hidden_channels,270 kernel_size,271 dilation_rate,272 n_layers,273 gin_channels=0):274 super().__init__()275 self.in_channels = in_channels276 self.out_channels = out_channels277 self.hidden_channels = hidden_channels278 self.kernel_size = kernel_size279 self.dilation_rate = dilation_rate280 self.n_layers = n_layers281 self.gin_channels = gin_channels282 283 self.pre = nn.Conv1d(in_channels, hidden_channels, 1)284 self.enc = modules.WN(hidden_channels,285 kernel_size,286 dilation_rate,287 n_layers,288 gin_channels=gin_channels)289 self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)290 291 def forward(self, x, x_lengths, g=None):292 x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)),293 1).to(x.dtype)294 x = self.pre(x) * x_mask295 x = self.enc(x, x_mask, g=g)296 stats = self.proj(x) * x_mask297 m, logs = torch.split(stats, self.out_channels, dim=1)298 z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask299 return z, m, logs, x_mask300 301 302class Generator(nn.Module):303 304 def __init__(self,305 initial_channel,306 resblock,307 resblock_kernel_sizes,308 resblock_dilation_sizes,309 upsample_rates,310 upsample_initial_channel,311 upsample_kernel_sizes,312 gin_channels=0):313 super(Generator, self).__init__()314 self.num_kernels = len(resblock_kernel_sizes)315 self.num_upsamples = len(upsample_rates)316 self.conv_pre = Conv1d(initial_channel,317 upsample_initial_channel,318 7,319 1,320 padding=3)321 resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2322 323 self.ups = nn.ModuleList()324 for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):325 self.ups.append(326 weight_norm(327 ConvTranspose1d(upsample_initial_channel // (2 ** i),328 upsample_initial_channel // (2 ** (i + 1)),329 k,330 u,331 padding=(k - u) // 2)))332 333 self.resblocks = nn.ModuleList()334 self.conv_posts = nn.ModuleList()335 for i in range(len(self.ups)):336 ch = upsample_initial_channel // (2 ** (i + 1))337 for j, (k, d) in enumerate(338 zip(resblock_kernel_sizes, resblock_dilation_sizes)):339 self.resblocks.append(resblock(ch, k, d))340 if i >= len(self.ups) - 3:341 self.conv_posts.append(342 Conv1d(ch, 1, 7, 1, padding=3, bias=False))343 self.ups.apply(init_weights)344 345 if gin_channels != 0:346 self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)347 348 def forward(self, x, g=None):349 x = self.conv_pre(x)350 if g is not None:351 x = x + self.cond(g)352 353 for i in range(self.num_upsamples):354 x = F.leaky_relu(x, modules.LRELU_SLOPE)355 x = self.ups[i](x)356 xs = None357 for j in range(self.num_kernels):358 xs = xs + self.resblocks[i * self.num_kernels + j](x) if xs is not None \359 else self.resblocks[i * self.num_kernels + j](x)360 x = xs / self.num_kernels361 x = F.leaky_relu(x)362 x = self.conv_posts[-1](x)363 x = torch.tanh(x)364 365 return x366 367 def hier_forward(self, x, g=None):368 outs = []369 x = self.conv_pre(x)370 if g is not None:371 x = x + self.cond(g)372 373 for i in range(self.num_upsamples):374 x = F.leaky_relu(x, modules.LRELU_SLOPE)375 x = self.ups[i](x)376 xs = None377 for j in range(self.num_kernels):378 xs = xs + self.resblocks[i * self.num_kernels + j](x) if xs is not None \379 else self.resblocks[i * self.num_kernels + j](x)380 x = xs / self.num_kernels381 if i >= self.num_upsamples - 3:382 _x = F.leaky_relu(x)383 _x = self.conv_posts[i - self.num_upsamples + 3](_x)384 _x = torch.tanh(_x)385 outs.append(_x)386 return outs387 388 def remove_weight_norm(self):389 print('Removing weight norm...')390 for l in self.ups:391 remove_weight_norm(l)392 for l in self.resblocks:393 l.remove_weight_norm()394 395 396class DiscriminatorP(nn.Module):397 398 def __init__(self,399 period,400 kernel_size=5,401 stride=3,402 use_spectral_norm=False):403 super(DiscriminatorP, self).__init__()404 self.period = period405 self.use_spectral_norm = use_spectral_norm406 norm_f = weight_norm if use_spectral_norm == False else spectral_norm407 self.convs = nn.ModuleList([408 norm_f(409 Conv2d(1,410 32, (kernel_size, 1), (stride, 1),411 padding=(get_padding(kernel_size, 1), 0))),412 norm_f(413 Conv2d(32,414 128, (kernel_size, 1), (stride, 1),415 padding=(get_padding(kernel_size, 1), 0))),416 norm_f(417 Conv2d(128,418 512, (kernel_size, 1), (stride, 1),419 padding=(get_padding(kernel_size, 1), 0))),420 norm_f(421 Conv2d(512,422 1024, (kernel_size, 1), (stride, 1),423 padding=(get_padding(kernel_size, 1), 0))),424 norm_f(425 Conv2d(1024,426 1024, (kernel_size, 1),427 1,428 padding=(get_padding(kernel_size, 1), 0))),429 ])430 self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))431 432 def forward(self, x):433 fmap = []434 435 # 1d to 2d436 b, c, t = x.shape437 if t % self.period != 0: # pad first438 n_pad = self.period - (t % self.period)439 x = F.pad(x, (0, n_pad), "reflect")440 t = t + n_pad441 x = x.view(b, c, t // self.period, self.period)442 443 for l in self.convs:444 x = l(x)445 x = F.leaky_relu(x, modules.LRELU_SLOPE)446 fmap.append(x)447 x = self.conv_post(x)448 fmap.append(x)449 x = torch.flatten(x, 1, -1)450 451 return x, fmap452 453 454class DiscriminatorS(nn.Module):455 456 def __init__(self, use_spectral_norm=False):457 super(DiscriminatorS, self).__init__()458 norm_f = weight_norm if use_spectral_norm == False else spectral_norm459 self.convs = nn.ModuleList([460 norm_f(Conv1d(1, 16, 15, 1, padding=7)),461 norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),462 norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),463 norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),464 norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),465 norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),466 ])467 self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))468 469 def forward(self, x):470 fmap = []471 472 for l in self.convs:473 x = l(x)474 x = F.leaky_relu(x, modules.LRELU_SLOPE)475 fmap.append(x)476 x = self.conv_post(x)477 fmap.append(x)478 x = torch.flatten(x, 1, -1)479 480 return x, fmap481 482 483class MultiPeriodDiscriminator(nn.Module):484 485 def __init__(self, use_spectral_norm=False):486 super(MultiPeriodDiscriminator, self).__init__()487 periods = [2, 3, 5, 7, 11]488 489 discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]490 discs = discs + \491 [DiscriminatorP(i, use_spectral_norm=use_spectral_norm)492 for i in periods]493 self.discriminators = nn.ModuleList(discs)494 495 def forward(self, y, y_hat):496 y_d_rs = []497 y_d_gs = []498 fmap_rs = []499 fmap_gs = []500 for i, d in enumerate(self.discriminators):501 y_d_r, fmap_r = d(y)502 y_d_g, fmap_g = d(y_hat)503 y_d_rs.append(y_d_r)504 y_d_gs.append(y_d_g)505 fmap_rs.append(fmap_r)506 fmap_gs.append(fmap_g)507 508 return y_d_rs, y_d_gs, fmap_rs, fmap_gs509 510 511##### Avocodo512class CoMBDBlock(torch.nn.Module):513 514 def __init__(515 self,516 h_u, # List[int],517 d_k, # List[int],518 d_s, # List[int],519 d_d, # List[int],520 d_g, # List[int],521 d_p, # List[int],522 op_f, # int,523 op_k, # int,524 op_g, # int,525 use_spectral_norm=False):526 super(CoMBDBlock, self).__init__()527 norm_f = weight_norm if use_spectral_norm is False else spectral_norm528 529 self.convs = nn.ModuleList()530 filters = [[1, h_u[0]]]531 for i in range(len(h_u) - 1):532 filters.append([h_u[i], h_u[i + 1]])533 for _f, _k, _s, _d, _g, _p in zip(filters, d_k, d_s, d_d, d_g, d_p):534 self.convs.append(535 norm_f(536 Conv1d(in_channels=_f[0],537 out_channels=_f[1],538 kernel_size=_k,539 stride=_s,540 dilation=_d,541 groups=_g,542 padding=_p)))543 self.projection_conv = norm_f(544 Conv1d(in_channels=filters[-1][1],545 out_channels=op_f,546 kernel_size=op_k,547 groups=op_g))548 549 def forward(self, x, b_y, b_y_hat):550 fmap_r = []551 fmap_g = []552 for block in self.convs:553 x = block(x)554 x = F.leaky_relu(x, 0.2)555 f_r, f_g = x.split([b_y, b_y_hat], dim=0)556 fmap_r.append(f_r.tile([2, 1, 1]) if b_y < b_y_hat else f_r)557 fmap_g.append(f_g)558 x = self.projection_conv(x)559 x_r, x_g = x.split([b_y, b_y_hat], dim=0)560 return x_r.tile([2, 1, 1561 ]) if b_y < b_y_hat else x_r, x_g, fmap_r, fmap_g562 563 564class CoMBD(torch.nn.Module):565 566 def __init__(self, use_spectral_norm=False):567 super(CoMBD, self).__init__()568 self.pqmf_list = nn.ModuleList([569 PQMF(4, 192, 0.13, 10.0), # lv2570 PQMF(2, 256, 0.25, 10.0) # lv1571 ])572 combd_h_u = [[16, 64, 256, 1024, 1024, 1024] for _ in range(3)]573 combd_d_k = [[7, 11, 11, 11, 11, 5], [11, 21, 21, 21, 21, 5],574 [15, 41, 41, 41, 41, 5]]575 combd_d_s = [[1, 1, 4, 4, 4, 1] for _ in range(3)]576 combd_d_d = [[1, 1, 1, 1, 1, 1] for _ in range(3)]577 combd_d_g = [[1, 4, 16, 64, 256, 1] for _ in range(3)]578 579 combd_d_p = [[3, 5, 5, 5, 5, 2], [5, 10, 10, 10, 10, 2],580 [7, 20, 20, 20, 20, 2]]581 combd_op_f = [1, 1, 1]582 combd_op_k = [3, 3, 3]583 combd_op_g = [1, 1, 1]584 585 self.blocks = nn.ModuleList()586 for _h_u, _d_k, _d_s, _d_d, _d_g, _d_p, _op_f, _op_k, _op_g in zip(587 combd_h_u,588 combd_d_k,589 combd_d_s,590 combd_d_d,591 combd_d_g,592 combd_d_p,593 combd_op_f,594 combd_op_k,595 combd_op_g,596 ):597 self.blocks.append(598 CoMBDBlock(599 _h_u,600 _d_k,601 _d_s,602 _d_d,603 _d_g,604 _d_p,605 _op_f,606 _op_k,607 _op_g,608 ))609 610 def _block_forward(self, ys, ys_hat, blocks):611 outs_real = []612 outs_fake = []613 f_maps_real = []614 f_maps_fake = []615 for y, y_hat, block in zip(ys, ys_hat,616 blocks): # y:B, y_hat: 2B if i!=-1 else B,B617 b_y = y.shape[0]618 b_y_hat = y_hat.shape[0]619 cat_y = torch.cat([y, y_hat], dim=0)620 out_real, out_fake, f_map_r, f_map_g = block(cat_y, b_y, b_y_hat)621 outs_real.append(out_real)622 outs_fake.append(out_fake)623 f_maps_real.append(f_map_r)624 f_maps_fake.append(f_map_g)625 return outs_real, outs_fake, f_maps_real, f_maps_fake626 627 def _pqmf_forward(self, ys, ys_hat):628 # preprocess for multi_scale forward629 multi_scale_inputs_hat = []630 for pqmf_ in self.pqmf_list:631 multi_scale_inputs_hat.append(pqmf_.analysis(ys_hat[-1])[:, :1, :])632 633 # real634 # for hierarchical forward635 # outs_real_, f_maps_real_ = self._block_forward(636 # ys, self.blocks)637 638 # for multi_scale forward639 # outs_real, f_maps_real = self._block_forward(640 # ys[:-1], self.blocks[:-1], outs_real, f_maps_real)641 # outs_real.extend(outs_real[:-1])642 # f_maps_real.extend(f_maps_real[:-1])643 644 # outs_real = [torch.cat([o,o], dim=0) if i!=len(outs_real_)-1 else o for i,o in enumerate(outs_real_)]645 # f_maps_real = [[torch.cat([fmap,fmap], dim=0) if i!=len(f_maps_real_)-1 else fmap for fmap in fmaps ] \646 # for i,fmaps in enumerate(f_maps_real_)]647 648 inputs_fake = [649 torch.cat([y, multi_scale_inputs_hat[i]], dim=0)650 if i != len(ys_hat) - 1 else y for i, y in enumerate(ys_hat)651 ]652 outs_real, outs_fake, f_maps_real, f_maps_fake = self._block_forward(653 ys, inputs_fake, self.blocks)654 655 # predicted656 # for hierarchical forward657 # outs_fake, f_maps_fake = self._block_forward(658 # inputs_fake, self.blocks)659 660 # outs_real_, f_maps_real_ = self._block_forward(661 # ys, self.blocks)662 # for multi_scale forward663 # outs_fake, f_maps_fake = self._block_forward(664 # multi_scale_inputs_hat, self.blocks[:-1], outs_fake, f_maps_fake)665 666 return outs_real, outs_fake, f_maps_real, f_maps_fake667 668 def forward(self, ys, ys_hat):669 outs_real, outs_fake, f_maps_real, f_maps_fake = self._pqmf_forward(670 ys, ys_hat)671 return outs_real, outs_fake, f_maps_real, f_maps_fake672 673 674class MDC(torch.nn.Module):675 676 def __init__(self,677 in_channels,678 out_channels,679 strides,680 kernel_size,681 dilations,682 use_spectral_norm=False):683 super(MDC, self).__init__()684 norm_f = weight_norm if not use_spectral_norm else spectral_norm685 self.d_convs = nn.ModuleList()686 for _k, _d in zip(kernel_size, dilations):687 self.d_convs.append(688 norm_f(689 Conv1d(in_channels=in_channels,690 out_channels=out_channels,691 kernel_size=_k,692 dilation=_d,693 padding=get_padding(_k, _d))))694 self.post_conv = norm_f(695 Conv1d(in_channels=out_channels,696 out_channels=out_channels,697 kernel_size=3,698 stride=strides,699 padding=get_padding(_k, _d)))700 self.softmax = torch.nn.Softmax(dim=-1)701 702 def forward(self, x):703 _out = None704 for _l in self.d_convs:705 _x = torch.unsqueeze(_l(x), -1)706 _x = F.leaky_relu(_x, 0.2)707 _out = torch.cat([_out, _x], axis=-1) if _out is not None \708 else _x709 x = torch.sum(_out, dim=-1)710 x = self.post_conv(x)711 x = F.leaky_relu(x, 0.2) # @@712 713 return x714 715 716class SBDBlock(torch.nn.Module):717 718 def __init__(self,719 segment_dim,720 strides,721 filters,722 kernel_size,723 dilations,724 use_spectral_norm=False):725 super(SBDBlock, self).__init__()726 norm_f = weight_norm if not use_spectral_norm else spectral_norm727 self.convs = nn.ModuleList()728 filters_in_out = [(segment_dim, filters[0])]729 for i in range(len(filters) - 1):730 filters_in_out.append([filters[i], filters[i + 1]])731 732 for _s, _f, _k, _d in zip(strides, filters_in_out, kernel_size,733 dilations):734 self.convs.append(735 MDC(in_channels=_f[0],736 out_channels=_f[1],737 strides=_s,738 kernel_size=_k,739 dilations=_d,740 use_spectral_norm=use_spectral_norm))741 self.post_conv = norm_f(742 Conv1d(in_channels=_f[1],743 out_channels=1,744 kernel_size=3,745 stride=1,746 padding=3 // 2)) # @@747 748 def forward(self, x):749 fmap_r = []750 fmap_g = []751 for _l in self.convs:752 x = _l(x)753 f_r, f_g = torch.chunk(x, 2, dim=0)754 fmap_r.append(f_r)755 fmap_g.append(f_g)756 x = self.post_conv(x) # @@757 x_r, x_g = torch.chunk(x, 2, dim=0)758 return x_r, x_g, fmap_r, fmap_g759 760 761class MDCDConfig:762 763 def __init__(self):764 self.pqmf_params = [16, 256, 0.03, 10.0]765 self.f_pqmf_params = [64, 256, 0.1, 9.0]766 self.filters = [[64, 128, 256, 256, 256], [64, 128, 256, 256, 256],767 [64, 128, 256, 256, 256], [32, 64, 128, 128, 128]]768 self.kernel_sizes = [[[7, 7, 7], [7, 7, 7], [7, 7, 7], [7, 7, 7],769 [7, 7, 7]],770 [[5, 5, 5], [5, 5, 5], [5, 5, 5], [5, 5, 5],771 [5, 5, 5]],772 [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3],773 [3, 3, 3]],774 [[5, 5, 5], [5, 5, 5], [5, 5, 5], [5, 5, 5],775 [5, 5, 5]]]776 self.dilations = [[[5, 7, 11], [5, 7, 11], [5, 7, 11], [5, 7, 11],777 [5, 7, 11]],778 [[3, 5, 7], [3, 5, 7], [3, 5, 7], [3, 5, 7],779 [3, 5, 7]],780 [[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3],781 [1, 2, 3]],782 [[1, 2, 3], [1, 2, 3], [1, 2, 3], [2, 3, 5],783 [2, 3, 5]]]784 self.strides = [[1, 1, 3, 3, 1], [1, 1, 3, 3, 1], [1, 1, 3, 3, 1],785 [1, 1, 3, 3, 1]]786 self.band_ranges = [[0, 6], [0, 11], [0, 16], [0, 64]]787 self.transpose = [False, False, False, True]788 self.segment_size = 8192789 790 791class SBD(torch.nn.Module):792 793 def __init__(self, use_spectral_norm=False):794 super(SBD, self).__init__()795 self.config = MDCDConfig()796 self.pqmf = PQMF(*self.config.pqmf_params)797 if True in self.config.transpose:798 self.f_pqmf = PQMF(*self.config.f_pqmf_params)799 else:800 self.f_pqmf = None801 802 self.discriminators = torch.nn.ModuleList()803 804 for _f, _k, _d, _s, _br, _tr in zip(self.config.filters,805 self.config.kernel_sizes,806 self.config.dilations,807 self.config.strides,808 self.config.band_ranges,809 self.config.transpose):810 if _tr:811 segment_dim = self.config.segment_size // _br[1] - _br[0]812 else:813 segment_dim = _br[1] - _br[0]814 815 self.discriminators.append(816 SBDBlock(segment_dim=segment_dim,817 filters=_f,818 kernel_size=_k,819 dilations=_d,820 strides=_s,821 use_spectral_norm=use_spectral_norm))822 823 def forward(self, y, y_hat):824 y_d_rs = []825 y_d_gs = []826 fmap_rs = []827 fmap_gs = []828 y_in = self.pqmf.analysis(y)829 y_hat_in = self.pqmf.analysis(y_hat)830 y_in_f = self.f_pqmf.analysis(y)831 y_hat_in_f = self.f_pqmf.analysis(y_hat)832 833 for d, br, tr in zip(self.discriminators, self.config.band_ranges,834 self.config.transpose):835 if not tr:836 _y_in = y_in[:, br[0]:br[1], :]837 _y_hat_in = y_hat_in[:, br[0]:br[1], :]838 else:839 _y_in = y_in_f[:, br[0]:br[1], :]840 _y_hat_in = y_hat_in_f[:, br[0]:br[1], :]841 _y_in = torch.transpose(_y_in, 1, 2)842 _y_hat_in = torch.transpose(_y_hat_in, 1, 2)843 # y_d_r, fmap_r = d(_y_in)844 # y_d_g, fmap_g = d(_y_hat_in)845 cat_y = torch.cat([_y_in, _y_hat_in], dim=0)846 y_d_r, y_d_g, fmap_r, fmap_g = d(cat_y)847 y_d_rs.append(y_d_r)848 fmap_rs.append(fmap_r)849 y_d_gs.append(y_d_g)850 fmap_gs.append(fmap_g)851 852 return y_d_rs, y_d_gs, fmap_rs, fmap_gs853 854 855class AvocodoDiscriminator(nn.Module):856 857 def __init__(self, use_spectral_norm=False):858 super(AvocodoDiscriminator, self).__init__()859 self.combd = CoMBD(use_spectral_norm)860 self.sbd = SBD(use_spectral_norm)861 862 def forward(self, y, ys_hat):863 ys = [864 self.combd.pqmf_list[0].analysis(y)[:, :1], # lv2865 self.combd.pqmf_list[1].analysis(y)[:, :1], # lv1866 y867 ]868 y_c_rs, y_c_gs, fmap_c_rs, fmap_c_gs = self.combd(ys, ys_hat)869 y_s_rs, y_s_gs, fmap_s_rs, fmap_s_gs = self.sbd(y, ys_hat[-1])870 y_c_rs.extend(y_s_rs)871 y_c_gs.extend(y_s_gs)872 fmap_c_rs.extend(fmap_s_rs)873 fmap_c_gs.extend(fmap_s_gs)874 return y_c_rs, y_c_gs, fmap_c_rs, fmap_c_gs875 876 877##### Avocodo878 879 880class YingDecoder(nn.Module):881 882 def __init__(self,883 hidden_channels,884 kernel_size,885 dilation_rate,886 n_layers,887 yin_start,888 yin_scope,889 yin_shift_range,890 gin_channels=0):891 super().__init__()892 self.in_channels = yin_scope893 self.out_channels = yin_scope894 self.hidden_channels = hidden_channels895 self.kernel_size = kernel_size896 self.dilation_rate = dilation_rate897 self.n_layers = n_layers898 self.gin_channels = gin_channels899 900 self.yin_start = yin_start901 self.yin_scope = yin_scope902 self.yin_shift_range = yin_shift_range903 904 self.pre = nn.Conv1d(self.in_channels, hidden_channels, 1)905 self.dec = modules.WN(hidden_channels,906 kernel_size,907 dilation_rate,908 n_layers,909 gin_channels=gin_channels)910 self.proj = nn.Conv1d(hidden_channels, self.out_channels, 1)911 912 def crop_scope(self, x, yin_start,913 scope_shift): # x: tensor [B,C,T] #scope_shift: tensor [B]914 return torch.stack([915 x[i, yin_start + scope_shift[i]:yin_start + self.yin_scope +916 scope_shift[i], :] for i in range(x.shape[0])917 ],918 dim=0)919 920 def infer(self, z_yin, z_mask, g=None):921 B = z_yin.shape[0]922 scope_shift = torch.randint(-self.yin_shift_range,923 self.yin_shift_range, (B,),924 dtype=torch.int)925 z_yin_crop = self.crop_scope(z_yin, self.yin_start, scope_shift)926 x = self.pre(z_yin_crop) * z_mask927 x = self.dec(x, z_mask, g=g)928 yin_hat_crop = self.proj(x) * z_mask929 return yin_hat_crop930 931 def forward(self, z_yin, yin_gt, z_mask, g=None):932 B = z_yin.shape[0]933 scope_shift = torch.randint(-self.yin_shift_range,934 self.yin_shift_range, (B,),935 dtype=torch.int)936 z_yin_crop = self.crop_scope(z_yin, self.yin_start, scope_shift)937 yin_gt_shifted_crop = self.crop_scope(yin_gt, self.yin_start,938 scope_shift)939 yin_gt_crop = self.crop_scope(yin_gt, self.yin_start,940 torch.zeros_like(scope_shift))941 x = self.pre(z_yin_crop) * z_mask942 x = self.dec(x, z_mask, g=g)943 yin_hat_crop = self.proj(x) * z_mask944 return yin_gt_crop, yin_gt_shifted_crop, yin_hat_crop, z_yin_crop, scope_shift945 946 947# For Q option948# class VQEmbedding(nn.Module):949#950# def __init__(self, codebook_size,951# code_channels):952# super().__init__()953# self.embedding = nn.Embedding(codebook_size, code_channels)954# self.embedding.weight.data.uniform_(-1. / codebook_size,955# 1. / codebook_size)956#957# def forward(self, z_e_x):958# z_e_x_ = z_e_x.permute(0, 2, 1).contiguous()959# latent_indices = vq(z_e_x_, self.embedding.weight)960# z_q = self.embedding(latent_indices).permute(0, 2, 1)961# return z_q962#963# def straight_through(self, z_e_x):964# z_e_x_ = z_e_x.permute(0, 2, 1).contiguous()965# z_q_x_st_, indices = vq_st(z_e_x_, self.embedding.weight.detach())966# z_q_x_st = z_q_x_st_.permute(0, 2, 1).contiguous()967#968# z_q_x_flatten = torch.index_select(self.embedding.weight,969# dim=0,970# index=indices)971# z_q_x_ = z_q_x_flatten.view_as(z_e_x_)972# z_q_x = z_q_x_.permute(0, 2, 1).contiguous()973# return z_q_x_st, z_q_x974 975 976class SynthesizerTrn(nn.Module):977 """978 Synthesizer for Training979 """980 981 def __init__(982 self,983 n_vocab,984 spec_channels,985 segment_size,986 midi_start,987 midi_end,988 octave_range,989 inter_channels,990 hidden_channels,991 filter_channels,992 n_heads,993 n_layers,994 kernel_size,995 p_dropout,996 resblock,997 resblock_kernel_sizes,998 resblock_dilation_sizes,999 upsample_rates,1000 upsample_initial_channel,1001 upsample_kernel_sizes,1002 yin_channels,1003 yin_start,1004 yin_scope,1005 yin_shift_range,1006 n_speakers=0,1007 gin_channels=0,1008 use_sdp=True,1009 # codebook_size=256, #for Q option1010 **kwargs):1011 1012 super().__init__()1013 self.n_vocab = n_vocab1014 self.spec_channels = spec_channels1015 self.inter_channels = inter_channels1016 self.hidden_channels = hidden_channels1017 self.filter_channels = filter_channels1018 self.n_heads = n_heads1019 self.n_layers = n_layers1020 self.kernel_size = kernel_size1021 self.p_dropout = p_dropout1022 self.resblock = resblock1023 self.resblock_kernel_sizes = resblock_kernel_sizes1024 self.resblock_dilation_sizes = resblock_dilation_sizes1025 self.upsample_rates = upsample_rates1026 self.upsample_initial_channel = upsample_initial_channel1027 self.upsample_kernel_sizes = upsample_kernel_sizes1028 self.segment_size = segment_size1029 self.n_speakers = n_speakers1030 self.gin_channels = gin_channels1031 1032 self.yin_channels = yin_channels1033 self.yin_start = yin_start1034 self.yin_scope = yin_scope1035 1036 self.use_sdp = use_sdp1037 self.enc_p = TextEncoder(n_vocab, inter_channels, hidden_channels,1038 filter_channels, n_heads, n_layers,1039 kernel_size, p_dropout)1040 self.dec = Generator(1041 inter_channels - yin_channels +1042 yin_scope,1043 resblock,1044 resblock_kernel_sizes,1045 resblock_dilation_sizes,1046 upsample_rates,1047 upsample_initial_channel,1048 upsample_kernel_sizes,1049 gin_channels=gin_channels)1050 1051 self.enc_spec = PosteriorEncoder(spec_channels,1052 inter_channels - yin_channels,1053 inter_channels - yin_channels,1054 5,1055 1,1056 16,1057 gin_channels=gin_channels)1058 1059 self.enc_pitch = PosteriorEncoder(yin_channels,1060 yin_channels,1061 yin_channels,1062 5,1063 1,1064 16,1065 gin_channels=gin_channels)1066 1067 self.flow = ResidualCouplingBlock(inter_channels,1068 hidden_channels,1069 5,1070 1,1071 4,1072 gin_channels=gin_channels)1073 1074 if use_sdp:1075 self.dp = StochasticDurationPredictor(hidden_channels,1076 192,1077 3,1078 0.5,1079 4,1080 gin_channels=gin_channels)1081 else:1082 self.dp = DurationPredictor(hidden_channels,1083 256,1084 3,1085 0.5,1086 gin_channels=gin_channels)1087 1088 self.yin_dec = YingDecoder(yin_scope,1089 5,1090 1,1091 4,1092 yin_start,1093 yin_scope,1094 yin_shift_range,1095 gin_channels=gin_channels)1096 1097 # self.vq = VQEmbedding(codebook_size, inter_channels - yin_channels)#inter_channels // 2)1098 self.emb_g = nn.Embedding(self.n_speakers, gin_channels)1099 1100 self.pitch = Pitch(midi_start=midi_start,1101 midi_end=midi_end,1102 octave_range=octave_range)1103 1104 def crop_scope(1105 self,1106 x,1107 scope_shift=0): # x: list #need to modify for non-scalar shift1108 return [1109 i[:, self.yin_start + scope_shift:self.yin_start + self.yin_scope +1110 scope_shift, :] for i in x1111 ]1112 1113 def crop_scope_tensor(1114 self, x,1115 scope_shift): # x: tensor [B,C,T] #scope_shift: tensor [B]1116 return torch.stack([1117 x[i, self.yin_start + scope_shift[i]:self.yin_start +1118 self.yin_scope + scope_shift[i], :] for i in range(x.shape[0])1119 ],1120 dim=0)1121 1122 def yin_dec_infer(self, z_yin, z_mask, sid=None):1123 if self.n_speakers > 0:1124 g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]1125 else:1126 g = None1127 return self.yin_dec.infer(z_yin, z_mask, g)1128 1129 def forward(self,1130 x,1131 t,1132 x_lengths,1133 y,1134 y_lengths,1135 ying,1136 ying_lengths,1137 sid=None,1138 scope_shift=0):1139 x, m_p, logs_p, x_mask = self.enc_p(x, t, x_lengths)1140 if self.n_speakers > 0:1141 g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]1142 else:1143 g = None1144 1145 z_spec, m_spec, logs_spec, spec_mask = self.enc_spec(y, y_lengths, g=g)1146 1147 # for Q option1148 # z_spec_q_st, z_spec_q = self.vq.straight_through(z_spec)1149 # z_spec_q_st = z_spec_q_st * spec_mask1150 # z_spec_q = z_spec_q * spec_mask1151 1152 z_yin, m_yin, logs_yin, yin_mask = self.enc_pitch(ying, y_lengths, g=g)1153 z_yin_crop, logs_yin_crop, m_yin_crop = self.crop_scope(1154 [z_yin, logs_yin, m_yin], scope_shift)1155 1156 # yin dec loss1157 yin_gt_crop, yin_gt_shifted_crop, yin_dec_crop, z_yin_crop_shifted, scope_shift = self.yin_dec(1158 z_yin, ying, yin_mask, g)1159 1160 z = torch.cat([z_spec, z_yin], dim=1)1161 logs_q = torch.cat([logs_spec, logs_yin], dim=1)1162 m_q = torch.cat([m_spec, m_yin], dim=1)1163 y_mask = spec_mask1164 1165 z_p = self.flow(z, y_mask, g=g)1166 1167 z_dec = torch.cat([z_spec, z_yin_crop], dim=1)1168 1169 z_dec_shifted = torch.cat([z_spec.detach(), z_yin_crop_shifted], dim=1)1170 z_dec_ = torch.cat([z_dec, z_dec_shifted], dim=0)1171 1172 with torch.no_grad():1173 # negative cross-entropy1174 s_p_sq_r = torch.exp(-2 * logs_p) # [b, d, t]1175 # [b, 1, t_s]1176 neg_cent1 = torch.sum(-0.5 * math.log(2 * math.pi) - logs_p, [1],1177 keepdim=True)1178 # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s], z_p: [b,d,t]1179 # neg_cent2 = torch.matmul(-0.5 * (z_p**2).transpose(1, 2), s_p_sq_r)1180 neg_cent2 = torch.einsum('bdt, bds -> bts', -0.5 * (z_p ** 2),1181 s_p_sq_r)1182 # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]1183 # neg_cent3 = torch.matmul(z_p.transpose(1, 2), (m_p * s_p_sq_r))1184 neg_cent3 = torch.einsum('bdt, bds -> bts', z_p, (m_p * s_p_sq_r))1185 neg_cent4 = torch.sum(-0.5 * (m_p ** 2) * s_p_sq_r, [1],1186 keepdim=True) # [b, 1, t_s]1187 neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent41188 1189 attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(1190 y_mask, -1)1191 from monotonic_align import maximum_path1192 attn = maximum_path(neg_cent,1193 attn_mask.squeeze(1)).unsqueeze(1).detach()1194 1195 w = attn.sum(2)1196 if self.use_sdp:1197 l_length = self.dp(x, x_mask, w, g=g)1198 l_length = l_length / torch.sum(x_mask)1199 else:1200 logw_ = torch.log(w + 1e-6) * x_mask