wonkitty/apple_oh
0
1"""20416后的更新：3 引入config中half4 重建npy而不用填写5 v2支持6 无f0模型支持7 修复8 9 int16：10 增加无索引支持11 f0算法改harvest(怎么看就只有这个会影响CPU占用)，但是不这么改效果不好12"""13import os, sys, traceback, re14 15import json16 17now_dir = os.getcwd()18sys.path.append(now_dir)19from configs.config import Config20 21Config = Config()22 23import torch_directml24import PySimpleGUI as sg25import sounddevice as sd26import noisereduce as nr27import numpy as np28from fairseq import checkpoint_utils29import librosa, torch, pyworld, faiss, time, threading30import torch.nn.functional as F31import torchaudio.transforms as tat32import scipy.signal as signal33 34 35# import matplotlib.pyplot as plt36from lib.infer_pack.models import (37 SynthesizerTrnMs256NSFsid,38 SynthesizerTrnMs256NSFsid_nono,39 SynthesizerTrnMs768NSFsid,40 SynthesizerTrnMs768NSFsid_nono,41)42from i18n import I18nAuto43 44i18n = I18nAuto()45device = torch_directml.device(torch_directml.default_device())46current_dir = os.getcwd()47 48 49class RVC:50 def __init__(51 self, key, hubert_path, pth_path, index_path, npy_path, index_rate52 ) -> None:53 """54 初始化55 """56 try:57 self.f0_up_key = key58 self.time_step = 160 / 16000 * 100059 self.f0_min = 5060 self.f0_max = 110061 self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)62 self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)63 self.sr = 1600064 self.window = 16065 if index_rate != 0:66 self.index = faiss.read_index(index_path)67 # self.big_npy = np.load(npy_path)68 self.big_npy = self.index.reconstruct_n(0, self.index.ntotal)69 print("index search enabled")70 self.index_rate = index_rate71 model_path = hubert_path72 print("load model(s) from {}".format(model_path))73 models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(74 [model_path],75 suffix="",76 )77 self.model = models[0]78 self.model = self.model.to(device)79 if Config.is_half:80 self.model = self.model.half()81 else:82 self.model = self.model.float()83 self.model.eval()84 cpt = torch.load(pth_path, map_location="cpu")85 self.tgt_sr = cpt["config"][-1]86 cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] # n_spk87 self.if_f0 = cpt.get("f0", 1)88 self.version = cpt.get("version", "v1")89 if self.version == "v1":90 if self.if_f0 == 1:91 self.net_g = SynthesizerTrnMs256NSFsid(92 *cpt["config"], is_half=Config.is_half93 )94 else:95 self.net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"])96 elif self.version == "v2":97 if self.if_f0 == 1:98 self.net_g = SynthesizerTrnMs768NSFsid(99 *cpt["config"], is_half=Config.is_half100 )101 else:102 self.net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"])103 del self.net_g.enc_q104 print(self.net_g.load_state_dict(cpt["weight"], strict=False))105 self.net_g.eval().to(device)106 if Config.is_half:107 self.net_g = self.net_g.half()108 else:109 self.net_g = self.net_g.float()110 except:111 print(traceback.format_exc())112 113 def get_f0(self, x, f0_up_key, inp_f0=None):114 x_pad = 1115 f0_min = 50116 f0_max = 1100117 f0_mel_min = 1127 * np.log(1 + f0_min / 700)118 f0_mel_max = 1127 * np.log(1 + f0_max / 700)119 f0, t = pyworld.harvest(120 x.astype(np.double),121 fs=self.sr,122 f0_ceil=f0_max,123 f0_floor=f0_min,124 frame_period=10,125 )126 f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.sr)127 f0 = signal.medfilt(f0, 3)128 f0 *= pow(2, f0_up_key / 12)129 # with open("test.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()]))130 tf0 = self.sr // self.window # 每秒f0点数131 if inp_f0 is not None:132 delta_t = np.round(133 (inp_f0[:, 0].max() - inp_f0[:, 0].min()) * tf0 + 1134 ).astype("int16")135 replace_f0 = np.interp(136 list(range(delta_t)), inp_f0[:, 0] * 100, inp_f0[:, 1]137 )138 shape = f0[x_pad * tf0 : x_pad * tf0 + len(replace_f0)].shape[0]139 f0[x_pad * tf0 : x_pad * tf0 + len(replace_f0)] = replace_f0[:shape]140 # with open("test_opt.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()]))141 f0bak = f0.copy()142 f0_mel = 1127 * np.log(1 + f0 / 700)143 f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (144 f0_mel_max - f0_mel_min145 ) + 1146 f0_mel[f0_mel <= 1] = 1147 f0_mel[f0_mel > 255] = 255148 f0_coarse = np.rint(f0_mel).astype(np.int)149 return f0_coarse, f0bak # 1-0150 151 def infer(self, feats: torch.Tensor) -> np.ndarray:152 """153 推理函数154 """155 audio = feats.clone().cpu().numpy()156 assert feats.dim() == 1, feats.dim()157 feats = feats.view(1, -1)158 padding_mask = torch.BoolTensor(feats.shape).fill_(False)159 if Config.is_half:160 feats = feats.half()161 else:162 feats = feats.float()163 inputs = {164 "source": feats.to(device),165 "padding_mask": padding_mask.to(device),166 "output_layer": 9 if self.version == "v1" else 12,167 }168 torch.cuda.synchronize()169 with torch.no_grad():170 logits = self.model.extract_features(**inputs)171 feats = (172 self.model.final_proj(logits[0]) if self.version == "v1" else logits[0]173 )174 175 ####索引优化176 try:177 if (178 hasattr(self, "index")179 and hasattr(self, "big_npy")180 and self.index_rate != 0181 ):182 npy = feats[0].cpu().numpy().astype("float32")183 score, ix = self.index.search(npy, k=8)184 weight = np.square(1 / score)185 weight /= weight.sum(axis=1, keepdims=True)186 npy = np.sum(self.big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)187 if Config.is_half:188 npy = npy.astype("float16")189 feats = (190 torch.from_numpy(npy).unsqueeze(0).to(device) * self.index_rate191 + (1 - self.index_rate) * feats192 )193 else:194 print("index search FAIL or disabled")195 except:196 traceback.print_exc()197 print("index search FAIL")198 feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1)199 torch.cuda.synchronize()200 print(feats.shape)201 if self.if_f0 == 1:202 pitch, pitchf = self.get_f0(audio, self.f0_up_key)203 p_len = min(feats.shape[1], 13000, pitch.shape[0]) # 太大了爆显存204 else:205 pitch, pitchf = None, None206 p_len = min(feats.shape[1], 13000) # 太大了爆显存207 torch.cuda.synchronize()208 # print(feats.shape,pitch.shape)209 feats = feats[:, :p_len, :]210 if self.if_f0 == 1:211 pitch = pitch[:p_len]212 pitchf = pitchf[:p_len]213 pitch = torch.LongTensor(pitch).unsqueeze(0).to(device)214 pitchf = torch.FloatTensor(pitchf).unsqueeze(0).to(device)215 p_len = torch.LongTensor([p_len]).to(device)216 ii = 0 # sid217 sid = torch.LongTensor([ii]).to(device)218 with torch.no_grad():219 if self.if_f0 == 1:220 infered_audio = (221 self.net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0]222 .data.cpu()223 .float()224 )225 else:226 infered_audio = (227 self.net_g.infer(feats, p_len, sid)[0][0, 0].data.cpu().float()228 )229 torch.cuda.synchronize()230 return infered_audio231 232 233class GUIConfig:234 def __init__(self) -> None:235 self.hubert_path: str = ""236 self.pth_path: str = ""237 self.index_path: str = ""238 self.npy_path: str = ""239 self.pitch: int = 12240 self.samplerate: int = 44100241 self.block_time: float = 1.0 # s242 self.buffer_num: int = 1243 self.threhold: int = -30244 self.crossfade_time: float = 0.08245 self.extra_time: float = 0.04246 self.I_noise_reduce = False247 self.O_noise_reduce = False248 self.index_rate = 0.3249 250 251class GUI:252 def __init__(self) -> None:253 self.config = GUIConfig()254 self.flag_vc = False255 256 self.launcher()257 258 def load(self):259 (260 input_devices,261 output_devices,262 input_devices_indices,263 output_devices_indices,264 ) = self.get_devices()265 try:266 with open("values1.json", "r") as j:267 data = json.load(j)268 except:269 with open("values1.json", "w") as j:270 data = {271 "pth_path": "",272 "index_path": "",273 "sg_input_device": input_devices[274 input_devices_indices.index(sd.default.device[0])275 ],276 "sg_output_device": output_devices[277 output_devices_indices.index(sd.default.device[1])278 ],279 "threhold": "-45",280 "pitch": "0",281 "index_rate": "0",282 "block_time": "1",283 "crossfade_length": "0.04",284 "extra_time": "1",285 }286 return data287 288 def launcher(self):289 data = self.load()290 sg.theme("LightBlue3")291 input_devices, output_devices, _, _ = self.get_devices()292 layout = [293 [294 sg.Frame(295 title=i18n("Load model"),296 layout=[297 [298 sg.Input(299 default_text="hubert_base.pt",300 key="hubert_path",301 disabled=True,302 ),303 sg.FileBrowse(304 i18n("Hubert Model"),305 initial_folder=os.path.join(os.getcwd()),306 file_types=(("pt files", "*.pt"),),307 ),308 ],309 [310 sg.Input(311 default_text=data.get("pth_path", ""),312 key="pth_path",313 ),314 sg.FileBrowse(315 i18n("Select the .pth file"),316 initial_folder=os.path.join(os.getcwd(), "weights"),317 file_types=(("weight files", "*.pth"),),318 ),319 ],320 [321 sg.Input(322 default_text=data.get("index_path", ""),323 key="index_path",324 ),325 sg.FileBrowse(326 i18n("Select the .index file"),327 initial_folder=os.path.join(os.getcwd(), "logs"),328 file_types=(("index files", "*.index"),),329 ),330 ],331 [332 sg.Input(333 default_text="你不需要填写这个You don't need write this.",334 key="npy_path",335 disabled=True,336 ),337 sg.FileBrowse(338 i18n("Select the .npy file"),339 initial_folder=os.path.join(os.getcwd(), "logs"),340 file_types=(("feature files", "*.npy"),),341 ),342 ],343 ],344 )345 ],346 [347 sg.Frame(348 layout=[349 [350 sg.Text(i18n("Input device")),351 sg.Combo(352 input_devices,353 key="sg_input_device",354 default_value=data.get("sg_input_device", ""),355 ),356 ],357 [358 sg.Text(i18n("Output device")),359 sg.Combo(360 output_devices,361 key="sg_output_device",362 default_value=data.get("sg_output_device", ""),363 ),364 ],365 ],366 title=i18n("Audio device (please use the same type of driver)"),367 )368 ],369 [370 sg.Frame(371 layout=[372 [373 sg.Text(i18n("Response threshold")),374 sg.Slider(375 range=(-60, 0),376 key="threhold",377 resolution=1,378 orientation="h",379 default_value=data.get("threhold", ""),380 ),381 ],382 [383 sg.Text(i18n("Pitch settings")),384 sg.Slider(385 range=(-24, 24),386 key="pitch",387 resolution=1,388 orientation="h",389 default_value=data.get("pitch", ""),390 ),391 ],392 [393 sg.Text(i18n("Index Rate")),394 sg.Slider(395 range=(0.0, 1.0),396 key="index_rate",397 resolution=0.01,398 orientation="h",399 default_value=data.get("index_rate", ""),400 ),401 ],402 ],403 title=i18n("General settings"),404 ),405 sg.Frame(406 layout=[407 [408 sg.Text(i18n("Sample length")),409 sg.Slider(410 range=(0.1, 3.0),411 key="block_time",412 resolution=0.1,413 orientation="h",414 default_value=data.get("block_time", ""),415 ),416 ],417 [418 sg.Text(i18n("Fade length")),419 sg.Slider(420 range=(0.01, 0.15),421 key="crossfade_length",422 resolution=0.01,423 orientation="h",424 default_value=data.get("crossfade_length", ""),425 ),426 ],427 [428 sg.Text(i18n("Extra推理时长")),429 sg.Slider(430 range=(0.05, 3.00),431 key="extra_time",432 resolution=0.01,433 orientation="h",434 default_value=data.get("extra_time", ""),435 ),436 ],437 [438 sg.Checkbox(i18n("Input noise reduction"), key="I_noise_reduce"),439 sg.Checkbox(i18n("Output noise reduction"), key="O_noise_reduce"),440 ],441 ],442 title=i18n("Performance settings"),443 ),444 ],445 [446 sg.Button(i18n("开始音频Convert"), key="start_vc"),447 sg.Button(i18n("停止音频Convert"), key="stop_vc"),448 sg.Text(i18n("Inference time (ms):")),449 sg.Text("0", key="infer_time"),450 ],451 ]452 self.window = sg.Window("RVC - GUI", layout=layout)453 self.event_handler()454 455 def event_handler(self):456 while True:457 event, values = self.window.read()458 if event == sg.WINDOW_CLOSED:459 self.flag_vc = False460 exit()461 if event == "start_vc" and self.flag_vc == False:462 if self.set_values(values) == True:463 print("using_cuda:" + str(torch.cuda.is_available()))464 self.start_vc()465 settings = {466 "pth_path": values["pth_path"],467 "index_path": values["index_path"],468 "sg_input_device": values["sg_input_device"],469 "sg_output_device": values["sg_output_device"],470 "threhold": values["threhold"],471 "pitch": values["pitch"],472 "index_rate": values["index_rate"],473 "block_time": values["block_time"],474 "crossfade_length": values["crossfade_length"],475 "extra_time": values["extra_time"],476 }477 with open("values1.json", "w") as j:478 json.dump(settings, j)479 if event == "stop_vc" and self.flag_vc == True:480 self.flag_vc = False481 482 def set_values(self, values):483 if len(values["pth_path"].strip()) == 0:484 sg.popup(i18n("Select the pth file"))485 return False486 if len(values["index_path"].strip()) == 0:487 sg.popup(i18n("Select the index file"))488 return False489 pattern = re.compile("[^\x00-\x7F]+")490 if pattern.findall(values["hubert_path"]):491 sg.popup(i18n("The hubert model path must not contain Chinese characters"))492 return False493 if pattern.findall(values["pth_path"]):494 sg.popup(i18n("The pth file path must not contain Chinese characters."))495 return False496 if pattern.findall(values["index_path"]):497 sg.popup(i18n("The index file path must not contain Chinese characters."))498 return False499 self.set_devices(values["sg_input_device"], values["sg_output_device"])500 self.config.hubert_path = os.path.join(current_dir, "hubert_base.pt")501 self.config.pth_path = values["pth_path"]502 self.config.index_path = values["index_path"]503 self.config.npy_path = values["npy_path"]504 self.config.threhold = values["threhold"]505 self.config.pitch = values["pitch"]506 self.config.block_time = values["block_time"]507 self.config.crossfade_time = values["crossfade_length"]508 self.config.extra_time = values["extra_time"]509 self.config.I_noise_reduce = values["I_noise_reduce"]510 self.config.O_noise_reduce = values["O_noise_reduce"]511 self.config.index_rate = values["index_rate"]512 return True513 514 def start_vc(self):515 torch.cuda.empty_cache()516 self.flag_vc = True517 self.block_frame = int(self.config.block_time * self.config.samplerate)518 self.crossfade_frame = int(self.config.crossfade_time * self.config.samplerate)519 self.sola_search_frame = int(0.012 * self.config.samplerate)520 self.delay_frame = int(0.01 * self.config.samplerate) # 往前预留0.02s521 self.extra_frame = int(self.config.extra_time * self.config.samplerate)522 self.rvc = None523 self.rvc = RVC(524 self.config.pitch,525 self.config.hubert_path,526 self.config.pth_path,527 self.config.index_path,528 self.config.npy_path,529 self.config.index_rate,530 )531 self.input_wav: np.ndarray = np.zeros(532 self.extra_frame533 + self.crossfade_frame534 + self.sola_search_frame535 + self.block_frame,536 dtype="float32",537 )538 self.output_wav: torch.Tensor = torch.zeros(539 self.block_frame, device=device, dtype=torch.float32540 )541 self.sola_buffer: torch.Tensor = torch.zeros(542 self.crossfade_frame, device=device, dtype=torch.float32543 )544 self.fade_in_window: torch.Tensor = torch.linspace(545 0.0, 1.0, steps=self.crossfade_frame, device=device, dtype=torch.float32546 )547 self.fade_out_window: torch.Tensor = 1 - self.fade_in_window548 self.resampler1 = tat.Resample(549 orig_freq=self.config.samplerate, new_freq=16000, dtype=torch.float32550 )551 self.resampler2 = tat.Resample(552 orig_freq=self.rvc.tgt_sr,553 new_freq=self.config.samplerate,554 dtype=torch.float32,555 )556 thread_vc = threading.Thread(target=self.soundinput)557 thread_vc.start()558 559 def soundinput(self):560 """561 接受音频输入562 """563 with sd.Stream(564 channels=2,565 callback=self.audio_callback,566 blocksize=self.block_frame,567 samplerate=self.config.samplerate,568 dtype="float32",569 ):570 while self.flag_vc:571 time.sleep(self.config.block_time)572 print("Audio block passed.")573 print("ENDing VC")574 575 def audio_callback(576 self, indata: np.ndarray, outdata: np.ndarray, frames, times, status577 ):578 """579 音频处理580 """581 start_time = time.perf_counter()582 indata = librosa.to_mono(indata.T)583 if self.config.I_noise_reduce:584 indata[:] = nr.reduce_noise(y=indata, sr=self.config.samplerate)585 586 """noise gate"""587 frame_length = 2048588 hop_length = 1024589 rms = librosa.feature.rms(590 y=indata, frame_length=frame_length, hop_length=hop_length591 )592 db_threhold = librosa.amplitude_to_db(rms, ref=1.0)[0] < self.config.threhold593 # print(rms.shape,db.shape,db)594 for i in range(db_threhold.shape[0]):595 if db_threhold[i]:596 indata[i * hop_length : (i + 1) * hop_length] = 0597 self.input_wav[:] = np.append(self.input_wav[self.block_frame :], indata)598 599 # infer600 print("input_wav:" + str(self.input_wav.shape))601 # print('infered_wav:'+str(infer_wav.shape))602 infer_wav: torch.Tensor = self.resampler2(603 self.rvc.infer(self.resampler1(torch.from_numpy(self.input_wav)))604 )[-self.crossfade_frame - self.sola_search_frame - self.block_frame :].to(605 device606 )607 print("infer_wav:" + str(infer_wav.shape))608 609 # SOLA algorithm from https://github.com/yxlllc/DDSP-SVC610 cor_nom = F.conv1d(611 infer_wav[None, None, : self.crossfade_frame + self.sola_search_frame],612 self.sola_buffer[None, None, :],613 )614 cor_den = torch.sqrt(615 F.conv1d(616 infer_wav[None, None, : self.crossfade_frame + self.sola_search_frame]617 ** 2,618 torch.ones(1, 1, self.crossfade_frame, device=device),619 )620 + 1e-8621 )622 sola_offset = torch.argmax(cor_nom[0, 0] / cor_den[0, 0])623 print("sola offset: " + str(int(sola_offset)))624 625 # crossfade626 self.output_wav[:] = infer_wav[sola_offset : sola_offset + self.block_frame]627 self.output_wav[: self.crossfade_frame] *= self.fade_in_window628 self.output_wav[: self.crossfade_frame] += self.sola_buffer[:]629 if sola_offset < self.sola_search_frame:630 self.sola_buffer[:] = (631 infer_wav[632 -self.sola_search_frame633 - self.crossfade_frame634 + sola_offset : -self.sola_search_frame635 + sola_offset636 ]637 * self.fade_out_window638 )639 else:640 self.sola_buffer[:] = (641 infer_wav[-self.crossfade_frame :] * self.fade_out_window642 )643 644 if self.config.O_noise_reduce:645 outdata[:] = np.tile(646 nr.reduce_noise(647 y=self.output_wav[:].cpu().numpy(), sr=self.config.samplerate648 ),649 (2, 1),650 ).T651 else:652 outdata[:] = self.output_wav[:].repeat(2, 1).t().cpu().numpy()653 total_time = time.perf_counter() - start_time654 self.window["infer_time"].update(int(total_time * 1000))655 print("infer time:" + str(total_time))656 657 def get_devices(self, update: bool = True):658 """获取设备列表"""659 if update:660 sd._terminate()661 sd._initialize()662 devices = sd.query_devices()663 hostapis = sd.query_hostapis()664 for hostapi in hostapis:665 for device_idx in hostapi["devices"]:666 devices[device_idx]["hostapi_name"] = hostapi["name"]667 input_devices = [668 f"{d['name']} ({d['hostapi_name']})"669 for d in devices670 if d["max_input_channels"] > 0671 ]672 output_devices = [673 f"{d['name']} ({d['hostapi_name']})"674 for d in devices675 if d["max_output_channels"] > 0676 ]677 input_devices_indices = [678 d["index"] if "index" in d else d["name"]679 for d in devices680 if d["max_input_channels"] > 0681 ]682 output_devices_indices = [683 d["index"] if "index" in d else d["name"]684 for d in devices685 if d["max_output_channels"] > 0686 ]687 return (688 input_devices,689 output_devices,690 input_devices_indices,691 output_devices_indices,692 )693 694 def set_devices(self, input_device, output_device):695 """设置输出设备"""696 (697 input_devices,698 output_devices,699 input_device_indices,700 output_device_indices,701 ) = self.get_devices()702 sd.default.device[0] = input_device_indices[input_devices.index(input_device)]703 sd.default.device[1] = output_device_indices[704 output_devices.index(output_device)705 ]706 print("input device:" + str(sd.default.device[0]) + ":" + str(input_device))707 print("output device:" + str(sd.default.device[1]) + ":" + str(output_device))708 709 710gui = GUI()711