surfmore/SimpleRVC
0
1import math2import torch3from torch import nn4from torch.nn import functional as F5from infer_pack import modules6from infer_pack import attentions7from infer_pack import commons8from infer_pack.commons import init_weights, get_padding9from torch.nn import Conv1d, ConvTranspose1d, Conv2d10from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm11from infer_pack.commons import init_weights12import numpy as np13 14class TextEncoder256(nn.Module):15 def __init__(16 self,17 out_channels,18 hidden_channels,19 filter_channels,20 n_heads,21 n_layers,22 kernel_size,23 p_dropout,24 f0=True,25 ):26 super().__init__()27 self.out_channels = out_channels28 self.hidden_channels = hidden_channels29 self.filter_channels = filter_channels30 self.n_heads = n_heads31 self.n_layers = n_layers32 self.kernel_size = kernel_size33 self.p_dropout = p_dropout34 self.emb_phone = nn.Linear(256, hidden_channels)35 self.lrelu = nn.LeakyReLU(0.1, inplace=True)36 if f0 == True:37 self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 25638 self.encoder = attentions.Encoder(39 hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout40 )41 self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)42 43 def forward(self, phone, pitch, lengths):44 if pitch == None:45 x = self.emb_phone(phone)46 else:47 x = self.emb_phone(phone) + self.emb_pitch(pitch)48 x = x * math.sqrt(self.hidden_channels) # [b, t, h]49 x = self.lrelu(x)50 x = torch.transpose(x, 1, -1) # [b, h, t]51 x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(52 x.dtype53 )54 x = self.encoder(x * x_mask, x_mask)55 stats = self.proj(x) * x_mask56 57 m, logs = torch.split(stats, self.out_channels, dim=1)58 return m, logs, x_mask59 60 61class TextEncoder768(nn.Module):62 def __init__(63 self,64 out_channels,65 hidden_channels,66 filter_channels,67 n_heads,68 n_layers,69 kernel_size,70 p_dropout,71 f0=True,72 ):73 super().__init__()74 self.out_channels = out_channels75 self.hidden_channels = hidden_channels76 self.filter_channels = filter_channels77 self.n_heads = n_heads78 self.n_layers = n_layers79 self.kernel_size = kernel_size80 self.p_dropout = p_dropout81 self.emb_phone = nn.Linear(768, hidden_channels)82 self.lrelu = nn.LeakyReLU(0.1, inplace=True)83 if f0 == True:84 self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 25685 self.encoder = attentions.Encoder(86 hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout87 )88 self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)89 90 def forward(self, phone, pitch, lengths):91 if pitch == None:92 x = self.emb_phone(phone)93 else:94 x = self.emb_phone(phone) + self.emb_pitch(pitch)95 x = x * math.sqrt(self.hidden_channels) # [b, t, h]96 x = self.lrelu(x)97 x = torch.transpose(x, 1, -1) # [b, h, t]98 x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(99 x.dtype100 )101 x = self.encoder(x * x_mask, x_mask)102 stats = self.proj(x) * x_mask103 104 m, logs = torch.split(stats, self.out_channels, dim=1)105 return m, logs, x_mask106 107 108class ResidualCouplingBlock(nn.Module):109 def __init__(110 self,111 channels,112 hidden_channels,113 kernel_size,114 dilation_rate,115 n_layers,116 n_flows=4,117 gin_channels=0,118 ):119 super().__init__()120 self.channels = channels121 self.hidden_channels = hidden_channels122 self.kernel_size = kernel_size123 self.dilation_rate = dilation_rate124 self.n_layers = n_layers125 self.n_flows = n_flows126 self.gin_channels = gin_channels127 128 self.flows = nn.ModuleList()129 for i in range(n_flows):130 self.flows.append(131 modules.ResidualCouplingLayer(132 channels,133 hidden_channels,134 kernel_size,135 dilation_rate,136 n_layers,137 gin_channels=gin_channels,138 mean_only=True,139 )140 )141 self.flows.append(modules.Flip())142 143 def forward(self, x, x_mask, g=None, reverse=False):144 if not reverse:145 for flow in self.flows:146 x, _ = flow(x, x_mask, g=g, reverse=reverse)147 else:148 for flow in reversed(self.flows):149 x = flow(x, x_mask, g=g, reverse=reverse)150 return x151 152 def remove_weight_norm(self):153 for i in range(self.n_flows):154 self.flows[i * 2].remove_weight_norm()155 156 157class PosteriorEncoder(nn.Module):158 def __init__(159 self,160 in_channels,161 out_channels,162 hidden_channels,163 kernel_size,164 dilation_rate,165 n_layers,166 gin_channels=0,167 ):168 super().__init__()169 self.in_channels = in_channels170 self.out_channels = out_channels171 self.hidden_channels = hidden_channels172 self.kernel_size = kernel_size173 self.dilation_rate = dilation_rate174 self.n_layers = n_layers175 self.gin_channels = gin_channels176 177 self.pre = nn.Conv1d(in_channels, hidden_channels, 1)178 self.enc = modules.WN(179 hidden_channels,180 kernel_size,181 dilation_rate,182 n_layers,183 gin_channels=gin_channels,184 )185 self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)186 187 def forward(self, x, x_lengths, g=None):188 x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(189 x.dtype190 )191 x = self.pre(x) * x_mask192 x = self.enc(x, x_mask, g=g)193 stats = self.proj(x) * x_mask194 m, logs = torch.split(stats, self.out_channels, dim=1)195 z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask196 return z, m, logs, x_mask197 198 def remove_weight_norm(self):199 self.enc.remove_weight_norm()200 201 202class Generator(torch.nn.Module):203 def __init__(204 self,205 initial_channel,206 resblock,207 resblock_kernel_sizes,208 resblock_dilation_sizes,209 upsample_rates,210 upsample_initial_channel,211 upsample_kernel_sizes,212 gin_channels=0,213 ):214 super(Generator, self).__init__()215 self.num_kernels = len(resblock_kernel_sizes)216 self.num_upsamples = len(upsample_rates)217 self.conv_pre = Conv1d(218 initial_channel, upsample_initial_channel, 7, 1, padding=3219 )220 resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2221 222 self.ups = nn.ModuleList()223 for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):224 self.ups.append(225 weight_norm(226 ConvTranspose1d(227 upsample_initial_channel // (2**i),228 upsample_initial_channel // (2 ** (i + 1)),229 k,230 u,231 padding=(k - u) // 2,232 )233 )234 )235 236 self.resblocks = nn.ModuleList()237 for i in range(len(self.ups)):238 ch = upsample_initial_channel // (2 ** (i + 1))239 for j, (k, d) in enumerate(240 zip(resblock_kernel_sizes, resblock_dilation_sizes)241 ):242 self.resblocks.append(resblock(ch, k, d))243 244 self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)245 self.ups.apply(init_weights)246 247 if gin_channels != 0:248 self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)249 250 def forward(self, x, g=None):251 x = self.conv_pre(x)252 if g is not None:253 x = x + self.cond(g)254 255 for i in range(self.num_upsamples):256 x = F.leaky_relu(x, modules.LRELU_SLOPE)257 x = self.ups[i](x)258 xs = None259 for j in range(self.num_kernels):260 if xs is None:261 xs = self.resblocks[i * self.num_kernels + j](x)262 else:263 xs += self.resblocks[i * self.num_kernels + j](x)264 x = xs / self.num_kernels265 x = F.leaky_relu(x)266 x = self.conv_post(x)267 x = torch.tanh(x)268 269 return x270 271 def remove_weight_norm(self):272 for l in self.ups:273 remove_weight_norm(l)274 for l in self.resblocks:275 l.remove_weight_norm()276 277 278class SineGen(torch.nn.Module):279 """Definition of sine generator280 SineGen(samp_rate, harmonic_num = 0,281 sine_amp = 0.1, noise_std = 0.003,282 voiced_threshold = 0,283 flag_for_pulse=False)284 samp_rate: sampling rate in Hz285 harmonic_num: number of harmonic overtones (default 0)286 sine_amp: amplitude of sine-wavefrom (default 0.1)287 noise_std: std of Gaussian noise (default 0.003)288 voiced_thoreshold: F0 threshold for U/V classification (default 0)289 flag_for_pulse: this SinGen is used inside PulseGen (default False)290 Note: when flag_for_pulse is True, the first time step of a voiced291 segment is always sin(np.pi) or cos(0)292 """293 294 def __init__(295 self,296 samp_rate,297 harmonic_num=0,298 sine_amp=0.1,299 noise_std=0.003,300 voiced_threshold=0,301 flag_for_pulse=False,302 ):303 super(SineGen, self).__init__()304 self.sine_amp = sine_amp305 self.noise_std = noise_std306 self.harmonic_num = harmonic_num307 self.dim = self.harmonic_num + 1308 self.sampling_rate = samp_rate309 self.voiced_threshold = voiced_threshold310 311 def _f02uv(self, f0):312 # generate uv signal313 uv = torch.ones_like(f0)314 uv = uv * (f0 > self.voiced_threshold)315 return uv316 317 def forward(self, f0, upp):318 """sine_tensor, uv = forward(f0)319 input F0: tensor(batchsize=1, length, dim=1)320 f0 for unvoiced steps should be 0321 output sine_tensor: tensor(batchsize=1, length, dim)322 output uv: tensor(batchsize=1, length, 1)323 """324 with torch.no_grad():325 f0 = f0[:, None].transpose(1, 2)326 f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, device=f0.device)327 # fundamental component328 f0_buf[:, :, 0] = f0[:, :, 0]329 for idx in np.arange(self.harmonic_num):330 f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (331 idx + 2332 ) # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic333 rad_values = (f0_buf / self.sampling_rate) % 1 ###%1意味着n_har的乘积无法后处理优化334 rand_ini = torch.rand(335 f0_buf.shape[0], f0_buf.shape[2], device=f0_buf.device336 )337 rand_ini[:, 0] = 0338 rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini339 tmp_over_one = torch.cumsum(rad_values, 1) # % 1 #####%1意味着后面的cumsum无法再优化340 tmp_over_one *= upp341 tmp_over_one = F.interpolate(342 tmp_over_one.transpose(2, 1),343 scale_factor=upp,344 mode="linear",345 align_corners=True,346 ).transpose(2, 1)347 rad_values = F.interpolate(348 rad_values.transpose(2, 1), scale_factor=upp, mode="nearest"349 ).transpose(350 2, 1351 ) #######352 tmp_over_one %= 1353 tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0354 cumsum_shift = torch.zeros_like(rad_values)355 cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0356 sine_waves = torch.sin(357 torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi358 )359 sine_waves = sine_waves * self.sine_amp360 uv = self._f02uv(f0)361 uv = F.interpolate(362 uv.transpose(2, 1), scale_factor=upp, mode="nearest"363 ).transpose(2, 1)364 noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3365 noise = noise_amp * torch.randn_like(sine_waves)366 sine_waves = sine_waves * uv + noise367 return sine_waves, uv, noise368 369 370class SourceModuleHnNSF(torch.nn.Module):371 """SourceModule for hn-nsf372 SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1,373 add_noise_std=0.003, voiced_threshod=0)374 sampling_rate: sampling_rate in Hz375 harmonic_num: number of harmonic above F0 (default: 0)376 sine_amp: amplitude of sine source signal (default: 0.1)377 add_noise_std: std of additive Gaussian noise (default: 0.003)378 note that amplitude of noise in unvoiced is decided379 by sine_amp380 voiced_threshold: threhold to set U/V given F0 (default: 0)381 Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)382 F0_sampled (batchsize, length, 1)383 Sine_source (batchsize, length, 1)384 noise_source (batchsize, length 1)385 uv (batchsize, length, 1)386 """387 388 def __init__(389 self,390 sampling_rate,391 harmonic_num=0,392 sine_amp=0.1,393 add_noise_std=0.003,394 voiced_threshod=0,395 is_half=True,396 ):397 super(SourceModuleHnNSF, self).__init__()398 399 self.sine_amp = sine_amp400 self.noise_std = add_noise_std401 self.is_half = is_half402 # to produce sine waveforms403 self.l_sin_gen = SineGen(404 sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod405 )406 407 # to merge source harmonics into a single excitation408 self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)409 self.l_tanh = torch.nn.Tanh()410 411 def forward(self, x, upp=None):412 sine_wavs, uv, _ = self.l_sin_gen(x, upp)413 if self.is_half:414 sine_wavs = sine_wavs.half()415 sine_merge = self.l_tanh(self.l_linear(sine_wavs))416 return sine_merge, None, None # noise, uv417 418 419class GeneratorNSF(torch.nn.Module):420 def __init__(421 self,422 initial_channel,423 resblock,424 resblock_kernel_sizes,425 resblock_dilation_sizes,426 upsample_rates,427 upsample_initial_channel,428 upsample_kernel_sizes,429 gin_channels,430 sr,431 is_half=False,432 ):433 super(GeneratorNSF, self).__init__()434 self.num_kernels = len(resblock_kernel_sizes)435 self.num_upsamples = len(upsample_rates)436 437 self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates))438 self.m_source = SourceModuleHnNSF(439 sampling_rate=sr, harmonic_num=0, is_half=is_half440 )441 self.noise_convs = nn.ModuleList()442 self.conv_pre = Conv1d(443 initial_channel, upsample_initial_channel, 7, 1, padding=3444 )445 resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2446 447 self.ups = nn.ModuleList()448 for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):449 c_cur = upsample_initial_channel // (2 ** (i + 1))450 self.ups.append(451 weight_norm(452 ConvTranspose1d(453 upsample_initial_channel // (2**i),454 upsample_initial_channel // (2 ** (i + 1)),455 k,456 u,457 padding=(k - u) // 2,458 )459 )460 )461 if i + 1 < len(upsample_rates):462 stride_f0 = np.prod(upsample_rates[i + 1 :])463 self.noise_convs.append(464 Conv1d(465 1,466 c_cur,467 kernel_size=stride_f0 * 2,468 stride=stride_f0,469 padding=stride_f0 // 2,470 )471 )472 else:473 self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))474 475 self.resblocks = nn.ModuleList()476 for i in range(len(self.ups)):477 ch = upsample_initial_channel // (2 ** (i + 1))478 for j, (k, d) in enumerate(479 zip(resblock_kernel_sizes, resblock_dilation_sizes)480 ):481 self.resblocks.append(resblock(ch, k, d))482 483 self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)484 self.ups.apply(init_weights)485 486 if gin_channels != 0:487 self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)488 489 self.upp = np.prod(upsample_rates)490 491 def forward(self, x, f0, g=None):492 har_source, noi_source, uv = self.m_source(f0, self.upp)493 har_source = har_source.transpose(1, 2)494 x = self.conv_pre(x)495 if g is not None:496 x = x + self.cond(g)497 498 for i in range(self.num_upsamples):499 x = F.leaky_relu(x, modules.LRELU_SLOPE)500 x = self.ups[i](x)501 x_source = self.noise_convs[i](har_source)502 x = x + x_source503 xs = None504 for j in range(self.num_kernels):505 if xs is None:506 xs = self.resblocks[i * self.num_kernels + j](x)507 else:508 xs += self.resblocks[i * self.num_kernels + j](x)509 x = xs / self.num_kernels510 x = F.leaky_relu(x)511 x = self.conv_post(x)512 x = torch.tanh(x)513 return x514 515 def remove_weight_norm(self):516 for l in self.ups:517 remove_weight_norm(l)518 for l in self.resblocks:519 l.remove_weight_norm()520 521 522sr2sr = {523 "32k": 32000,524 "40k": 40000,525 "48k": 48000,526}527 528 529class SynthesizerTrnMs256NSFsid(nn.Module):530 def __init__(531 self,532 spec_channels,533 segment_size,534 inter_channels,535 hidden_channels,536 filter_channels,537 n_heads,538 n_layers,539 kernel_size,540 p_dropout,541 resblock,542 resblock_kernel_sizes,543 resblock_dilation_sizes,544 upsample_rates,545 upsample_initial_channel,546 upsample_kernel_sizes,547 spk_embed_dim,548 gin_channels,549 sr,550 **kwargs551 ):552 super().__init__()553 if type(sr) == type("strr"):554 sr = sr2sr[sr]555 self.spec_channels = spec_channels556 self.inter_channels = inter_channels557 self.hidden_channels = hidden_channels558 self.filter_channels = filter_channels559 self.n_heads = n_heads560 self.n_layers = n_layers561 self.kernel_size = kernel_size562 self.p_dropout = p_dropout563 self.resblock = resblock564 self.resblock_kernel_sizes = resblock_kernel_sizes565 self.resblock_dilation_sizes = resblock_dilation_sizes566 self.upsample_rates = upsample_rates567 self.upsample_initial_channel = upsample_initial_channel568 self.upsample_kernel_sizes = upsample_kernel_sizes569 self.segment_size = segment_size570 self.gin_channels = gin_channels571 # self.hop_length = hop_length#572 self.spk_embed_dim = spk_embed_dim573 self.enc_p = TextEncoder256(574 inter_channels,575 hidden_channels,576 filter_channels,577 n_heads,578 n_layers,579 kernel_size,580 p_dropout,581 )582 self.dec = GeneratorNSF(583 inter_channels,584 resblock,585 resblock_kernel_sizes,586 resblock_dilation_sizes,587 upsample_rates,588 upsample_initial_channel,589 upsample_kernel_sizes,590 gin_channels=gin_channels,591 sr=sr,592 is_half=kwargs["is_half"],593 )594 self.enc_q = PosteriorEncoder(595 spec_channels,596 inter_channels,597 hidden_channels,598 5,599 1,600 16,601 gin_channels=gin_channels,602 )603 self.flow = ResidualCouplingBlock(604 inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels605 )606 self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)607 print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim)608 609 def remove_weight_norm(self):610 self.dec.remove_weight_norm()611 self.flow.remove_weight_norm()612 self.enc_q.remove_weight_norm()613 614 def forward(615 self, phone, phone_lengths, pitch, pitchf, y, y_lengths, ds616 ): # 这里ds是id，[bs,1]617 # print(1,pitch.shape)#[bs,t]618 g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的619 m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)620 z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)621 z_p = self.flow(z, y_mask, g=g)622 z_slice, ids_slice = commons.rand_slice_segments(623 z, y_lengths, self.segment_size624 )625 # print(-1,pitchf.shape,ids_slice,self.segment_size,self.hop_length,self.segment_size//self.hop_length)626 pitchf = commons.slice_segments2(pitchf, ids_slice, self.segment_size)627 # print(-2,pitchf.shape,z_slice.shape)628 o = self.dec(z_slice, pitchf, g=g)629 return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)630 631 def infer(self, phone, phone_lengths, pitch, nsff0, sid, rate=None):632 g = self.emb_g(sid).unsqueeze(-1)633 m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)634 z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask635 if rate:636 head = int(z_p.shape[2] * rate)637 z_p = z_p[:, :, -head:]638 x_mask = x_mask[:, :, -head:]639 nsff0 = nsff0[:, -head:]640 z = self.flow(z_p, x_mask, g=g, reverse=True)641 o = self.dec(z * x_mask, nsff0, g=g)642 return o, x_mask, (z, z_p, m_p, logs_p)643 644 645class SynthesizerTrnMs768NSFsid(nn.Module):646 def __init__(647 self,648 spec_channels,649 segment_size,650 inter_channels,651 hidden_channels,652 filter_channels,653 n_heads,654 n_layers,655 kernel_size,656 p_dropout,657 resblock,658 resblock_kernel_sizes,659 resblock_dilation_sizes,660 upsample_rates,661 upsample_initial_channel,662 upsample_kernel_sizes,663 spk_embed_dim,664 gin_channels,665 sr,666 **kwargs667 ):668 super().__init__()669 if type(sr) == type("strr"):670 sr = sr2sr[sr]671 self.spec_channels = spec_channels672 self.inter_channels = inter_channels673 self.hidden_channels = hidden_channels674 self.filter_channels = filter_channels675 self.n_heads = n_heads676 self.n_layers = n_layers677 self.kernel_size = kernel_size678 self.p_dropout = p_dropout679 self.resblock = resblock680 self.resblock_kernel_sizes = resblock_kernel_sizes681 self.resblock_dilation_sizes = resblock_dilation_sizes682 self.upsample_rates = upsample_rates683 self.upsample_initial_channel = upsample_initial_channel684 self.upsample_kernel_sizes = upsample_kernel_sizes685 self.segment_size = segment_size686 self.gin_channels = gin_channels687 # self.hop_length = hop_length#688 self.spk_embed_dim = spk_embed_dim689 self.enc_p = TextEncoder768(690 inter_channels,691 hidden_channels,692 filter_channels,693 n_heads,694 n_layers,695 kernel_size,696 p_dropout,697 )698 self.dec = GeneratorNSF(699 inter_channels,700 resblock,701 resblock_kernel_sizes,702 resblock_dilation_sizes,703 upsample_rates,704 upsample_initial_channel,705 upsample_kernel_sizes,706 gin_channels=gin_channels,707 sr=sr,708 is_half=kwargs["is_half"],709 )710 self.enc_q = PosteriorEncoder(711 spec_channels,712 inter_channels,713 hidden_channels,714 5,715 1,716 16,717 gin_channels=gin_channels,718 )719 self.flow = ResidualCouplingBlock(720 inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels721 )722 self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)723 print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim)724 725 def remove_weight_norm(self):726 self.dec.remove_weight_norm()727 self.flow.remove_weight_norm()728 self.enc_q.remove_weight_norm()729 730 def forward(731 self, phone, phone_lengths, pitch, pitchf, y, y_lengths, ds732 ): # 这里ds是id，[bs,1]733 # print(1,pitch.shape)#[bs,t]734 g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的735 m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)736 z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)737 z_p = self.flow(z, y_mask, g=g)738 z_slice, ids_slice = commons.rand_slice_segments(739 z, y_lengths, self.segment_size740 )741 # print(-1,pitchf.shape,ids_slice,self.segment_size,self.hop_length,self.segment_size//self.hop_length)742 pitchf = commons.slice_segments2(pitchf, ids_slice, self.segment_size)743 # print(-2,pitchf.shape,z_slice.shape)744 o = self.dec(z_slice, pitchf, g=g)745 return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)746 747 def infer(self, phone, phone_lengths, pitch, nsff0, sid, rate=None):748 g = self.emb_g(sid).unsqueeze(-1)749 m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)750 z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask751 if rate:752 head = int(z_p.shape[2] * rate)753 z_p = z_p[:, :, -head:]754 x_mask = x_mask[:, :, -head:]755 nsff0 = nsff0[:, -head:]756 z = self.flow(z_p, x_mask, g=g, reverse=True)757 o = self.dec(z * x_mask, nsff0, g=g)758 return o, x_mask, (z, z_p, m_p, logs_p)759 760 761class SynthesizerTrnMs256NSFsid_nono(nn.Module):762 def __init__(763 self,764 spec_channels,765 segment_size,766 inter_channels,767 hidden_channels,768 filter_channels,769 n_heads,770 n_layers,771 kernel_size,772 p_dropout,773 resblock,774 resblock_kernel_sizes,775 resblock_dilation_sizes,776 upsample_rates,777 upsample_initial_channel,778 upsample_kernel_sizes,779 spk_embed_dim,780 gin_channels,781 sr=None,782 **kwargs783 ):784 super().__init__()785 self.spec_channels = spec_channels786 self.inter_channels = inter_channels787 self.hidden_channels = hidden_channels788 self.filter_channels = filter_channels789 self.n_heads = n_heads790 self.n_layers = n_layers791 self.kernel_size = kernel_size792 self.p_dropout = p_dropout793 self.resblock = resblock794 self.resblock_kernel_sizes = resblock_kernel_sizes795 self.resblock_dilation_sizes = resblock_dilation_sizes796 self.upsample_rates = upsample_rates797 self.upsample_initial_channel = upsample_initial_channel798 self.upsample_kernel_sizes = upsample_kernel_sizes799 self.segment_size = segment_size800 self.gin_channels = gin_channels801 # self.hop_length = hop_length#802 self.spk_embed_dim = spk_embed_dim803 self.enc_p = TextEncoder256(804 inter_channels,805 hidden_channels,806 filter_channels,807 n_heads,808 n_layers,809 kernel_size,810 p_dropout,811 f0=False,812 )813 self.dec = Generator(814 inter_channels,815 resblock,816 resblock_kernel_sizes,817 resblock_dilation_sizes,818 upsample_rates,819 upsample_initial_channel,820 upsample_kernel_sizes,821 gin_channels=gin_channels,822 )823 self.enc_q = PosteriorEncoder(824 spec_channels,825 inter_channels,826 hidden_channels,827 5,828 1,829 16,830 gin_channels=gin_channels,831 )832 self.flow = ResidualCouplingBlock(833 inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels834 )835 self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)836 print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim)837 838 def remove_weight_norm(self):839 self.dec.remove_weight_norm()840 self.flow.remove_weight_norm()841 self.enc_q.remove_weight_norm()842 843 def forward(self, phone, phone_lengths, y, y_lengths, ds): # 这里ds是id，[bs,1]844 g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的845 m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)846 z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)847 z_p = self.flow(z, y_mask, g=g)848 z_slice, ids_slice = commons.rand_slice_segments(849 z, y_lengths, self.segment_size850 )851 o = self.dec(z_slice, g=g)852 return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)853 854 def infer(self, phone, phone_lengths, sid, rate=None):855 g = self.emb_g(sid).unsqueeze(-1)856 m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)857 z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask858 if rate:859 head = int(z_p.shape[2] * rate)860 z_p = z_p[:, :, -head:]861 x_mask = x_mask[:, :, -head:]862 z = self.flow(z_p, x_mask, g=g, reverse=True)863 o = self.dec(z * x_mask, g=g)864 return o, x_mask, (z, z_p, m_p, logs_p)865 866 867class SynthesizerTrnMs768NSFsid_nono(nn.Module):868 def __init__(869 self,870 spec_channels,871 segment_size,872 inter_channels,873 hidden_channels,874 filter_channels,875 n_heads,876 n_layers,877 kernel_size,878 p_dropout,879 resblock,880 resblock_kernel_sizes,881 resblock_dilation_sizes,882 upsample_rates,883 upsample_initial_channel,884 upsample_kernel_sizes,885 spk_embed_dim,886 gin_channels,887 sr=None,888 **kwargs889 ):890 super().__init__()891 self.spec_channels = spec_channels892 self.inter_channels = inter_channels893 self.hidden_channels = hidden_channels894 self.filter_channels = filter_channels895 self.n_heads = n_heads896 self.n_layers = n_layers897 self.kernel_size = kernel_size898 self.p_dropout = p_dropout899 self.resblock = resblock900 self.resblock_kernel_sizes = resblock_kernel_sizes901 self.resblock_dilation_sizes = resblock_dilation_sizes902 self.upsample_rates = upsample_rates903 self.upsample_initial_channel = upsample_initial_channel904 self.upsample_kernel_sizes = upsample_kernel_sizes905 self.segment_size = segment_size906 self.gin_channels = gin_channels907 # self.hop_length = hop_length#908 self.spk_embed_dim = spk_embed_dim909 self.enc_p = TextEncoder768(910 inter_channels,911 hidden_channels,912 filter_channels,913 n_heads,914 n_layers,915 kernel_size,916 p_dropout,917 f0=False,918 )919 self.dec = Generator(920 inter_channels,921 resblock,922 resblock_kernel_sizes,923 resblock_dilation_sizes,924 upsample_rates,925 upsample_initial_channel,926 upsample_kernel_sizes,927 gin_channels=gin_channels,928 )929 self.enc_q = PosteriorEncoder(930 spec_channels,931 inter_channels,932 hidden_channels,933 5,934 1,935 16,936 gin_channels=gin_channels,937 )938 self.flow = ResidualCouplingBlock(939 inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels940 )941 self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)942 print("gin_channels:", gin_channels, "self.spk_embed_dim:", self.spk_embed_dim)943 944 def remove_weight_norm(self):945 self.dec.remove_weight_norm()946 self.flow.remove_weight_norm()947 self.enc_q.remove_weight_norm()948 949 def forward(self, phone, phone_lengths, y, y_lengths, ds): # 这里ds是id，[bs,1]950 g = self.emb_g(ds).unsqueeze(-1) # [b, 256, 1]##1是t，广播的951 m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)952 z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)953 z_p = self.flow(z, y_mask, g=g)954 z_slice, ids_slice = commons.rand_slice_segments(955 z, y_lengths, self.segment_size956 )957 o = self.dec(z_slice, g=g)958 return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)959 960 def infer(self, phone, phone_lengths, sid, rate=None):961 g = self.emb_g(sid).unsqueeze(-1)962 m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)963 z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask964 if rate:965 head = int(z_p.shape[2] * rate)966 z_p = z_p[:, :, -head:]967 x_mask = x_mask[:, :, -head:]968 z = self.flow(z_p, x_mask, g=g, reverse=True)969 o = self.dec(z * x_mask, g=g)970 return o, x_mask, (z, z_p, m_p, logs_p)971 972 973class MultiPeriodDiscriminator(torch.nn.Module):974 def __init__(self, use_spectral_norm=False):975 super(MultiPeriodDiscriminator, self).__init__()976 periods = [2, 3, 5, 7, 11, 17]977 # periods = [3, 5, 7, 11, 17, 23, 37]978 979 discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]980 discs = discs + [981 DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods982 ]983 self.discriminators = nn.ModuleList(discs)984 985 def forward(self, y, y_hat):986 y_d_rs = [] #987 y_d_gs = []988 fmap_rs = []989 fmap_gs = []990 for i, d in enumerate(self.discriminators):991 y_d_r, fmap_r = d(y)992 y_d_g, fmap_g = d(y_hat)993 # for j in range(len(fmap_r)):994 # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape)995 y_d_rs.append(y_d_r)996 y_d_gs.append(y_d_g)997 fmap_rs.append(fmap_r)998 fmap_gs.append(fmap_g)999 1000 return y_d_rs, y_d_gs, fmap_rs, fmap_gs1001 1002 1003class MultiPeriodDiscriminatorV2(torch.nn.Module):1004 def __init__(self, use_spectral_norm=False):1005 super(MultiPeriodDiscriminatorV2, self).__init__()1006 # periods = [2, 3, 5, 7, 11, 17]1007 periods = [2, 3, 5, 7, 11, 17, 23, 37]1008 1009 discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]1010 discs = discs + [1011 DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods1012 ]1013 self.discriminators = nn.ModuleList(discs)1014 1015 def forward(self, y, y_hat):1016 y_d_rs = [] #1017 y_d_gs = []1018 fmap_rs = []1019 fmap_gs = []1020 for i, d in enumerate(self.discriminators):1021 y_d_r, fmap_r = d(y)1022 y_d_g, fmap_g = d(y_hat)1023 # for j in range(len(fmap_r)):1024 # print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape)1025 y_d_rs.append(y_d_r)1026 y_d_gs.append(y_d_g)1027 fmap_rs.append(fmap_r)1028 fmap_gs.append(fmap_g)1029 1030 return y_d_rs, y_d_gs, fmap_rs, fmap_gs1031 1032 1033class DiscriminatorS(torch.nn.Module):1034 def __init__(self, use_spectral_norm=False):1035 super(DiscriminatorS, self).__init__()1036 norm_f = weight_norm if use_spectral_norm == False else spectral_norm1037 self.convs = nn.ModuleList(1038 [1039 norm_f(Conv1d(1, 16, 15, 1, padding=7)),1040 norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),1041 norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),1042 norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),1043 norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),1044 norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),1045 ]1046 )1047 self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))1048 1049 def forward(self, x):1050 fmap = []1051 1052 for l in self.convs:1053 x = l(x)1054 x = F.leaky_relu(x, modules.LRELU_SLOPE)1055 fmap.append(x)1056 x = self.conv_post(x)1057 fmap.append(x)1058 x = torch.flatten(x, 1, -1)1059 1060 return x, fmap1061 1062 1063class DiscriminatorP(torch.nn.Module):1064 def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):1065 super(DiscriminatorP, self).__init__()1066 self.period = period1067 self.use_spectral_norm = use_spectral_norm1068 norm_f = weight_norm if use_spectral_norm == False else spectral_norm1069 self.convs = nn.ModuleList(1070 [1071 norm_f(1072 Conv2d(1073 1,1074 32,1075 (kernel_size, 1),1076 (stride, 1),1077 padding=(get_padding(kernel_size, 1), 0),1078 )1079 ),1080 norm_f(1081 Conv2d(1082 32,1083 128,1084 (kernel_size, 1),1085 (stride, 1),1086 padding=(get_padding(kernel_size, 1), 0),1087 )1088 ),1089 norm_f(1090 Conv2d(1091 128,1092 512,1093 (kernel_size, 1),1094 (stride, 1),1095 padding=(get_padding(kernel_size, 1), 0),1096 )1097 ),1098 norm_f(1099 Conv2d(1100 512,1101 1024,1102 (kernel_size, 1),1103 (stride, 1),1104 padding=(get_padding(kernel_size, 1), 0),1105 )1106 ),1107 norm_f(1108 Conv2d(1109 1024,1110 1024,1111 (kernel_size, 1),1112 1,1113 padding=(get_padding(kernel_size, 1), 0),1114 )1115 ),1116 ]1117 )1118 self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))1119 1120 def forward(self, x):1121 fmap = []1122 1123 # 1d to 2d1124 b, c, t = x.shape1125 if t % self.period != 0: # pad first1126 n_pad = self.period - (t % self.period)1127 x = F.pad(x, (0, n_pad), "reflect")1128 t = t + n_pad1129 x = x.view(b, c, t // self.period, self.period)1130 1131 for l in self.convs:1132 x = l(x)1133 x = F.leaky_relu(x, modules.LRELU_SLOPE)1134 fmap.append(x)1135 x = self.conv_post(x)1136 fmap.append(x)1137 x = torch.flatten(x, 1, -1)1138 1139 return x, fmap1140