goathead777/Zero_Shot_Inference
0
1# Modified from https://github.com/RVC-Boss/GPT-SoVITS/blob/main/GPT_SoVITS/inference_webui.py2import os3 4gpt_path = os.environ.get(5 "gpt_path", "pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt"6)7sovits_path = os.environ.get("sovits_path", "pretrained_models/s2G488k.pth")8cnhubert_base_path = os.environ.get(9 "cnhubert_base_path", "pretrained_models/chinese-hubert-base"10)11bert_path = os.environ.get(12 "bert_path", "pretrained_models/chinese-roberta-wwm-ext-large"13)14 15if "_CUDA_VISIBLE_DEVICES" in os.environ:16 os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"]17 18 19import gradio as gr20import librosa21import numpy as np22import torch23from transformers import AutoModelForMaskedLM, AutoTokenizer24 25from feature_extractor import cnhubert26 27cnhubert.cnhubert_base_path = cnhubert_base_path28from time import time as ttime29import datetime30 31from AR.models.t2s_lightning_module import Text2SemanticLightningModule32from module.mel_processing import spectrogram_torch33from module.models import SynthesizerTrn34from my_utils import load_audio35from text import cleaned_text_to_sequence36from text.cleaner import clean_text37 38device = "cuda" if torch.cuda.is_available() else "cpu"39 40is_half = eval(41 os.environ.get("is_half", "True" if torch.cuda.is_available() else "False")42)43 44tokenizer = AutoTokenizer.from_pretrained(bert_path)45bert_model = AutoModelForMaskedLM.from_pretrained(bert_path)46if is_half == True:47 bert_model = bert_model.half().to(device)48else:49 bert_model = bert_model.to(device)50 51 52# bert_model=bert_model.to(device)53def get_bert_feature(text, word2ph):54 with torch.no_grad():55 inputs = tokenizer(text, return_tensors="pt")56 for i in inputs:57 inputs[i] = inputs[i].to(device) #####输入是long不用管精度问题，精度随bert_model58 res = bert_model(**inputs, output_hidden_states=True)59 res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1]60 assert len(word2ph) == len(text)61 phone_level_feature = []62 for i in range(len(word2ph)):63 repeat_feature = res[i].repeat(word2ph[i], 1)64 phone_level_feature.append(repeat_feature)65 phone_level_feature = torch.cat(phone_level_feature, dim=0)66 # if(is_half==True):phone_level_feature=phone_level_feature.half()67 return phone_level_feature.T68 69 70n_semantic = 102471dict_s2 = torch.load(sovits_path, map_location="cpu")72hps = dict_s2["config"]73 74 75class DictToAttrRecursive:76 def __init__(self, input_dict):77 for key, value in input_dict.items():78 if isinstance(value, dict):79 # 如果值是字典，递归调用构造函数80 setattr(self, key, DictToAttrRecursive(value))81 else:82 setattr(self, key, value)83 84 85hps = DictToAttrRecursive(hps)86hps.model.semantic_frame_rate = "25hz"87dict_s1 = torch.load(gpt_path, map_location="cpu")88config = dict_s1["config"]89ssl_model = cnhubert.get_model()90if is_half == True:91 ssl_model = ssl_model.half().to(device)92else:93 ssl_model = ssl_model.to(device)94 95vq_model = SynthesizerTrn(96 hps.data.filter_length // 2 + 1,97 hps.train.segment_size // hps.data.hop_length,98 n_speakers=hps.data.n_speakers,99 **hps.model,100)101if is_half == True:102 vq_model = vq_model.half().to(device)103else:104 vq_model = vq_model.to(device)105vq_model.eval()106print(vq_model.load_state_dict(dict_s2["weight"], strict=False))107hz = 50108max_sec = config["data"]["max_sec"]109# t2s_model = Text2SemanticLightningModule.load_from_checkpoint(checkpoint_path=gpt_path, config=config, map_location="cpu")#########todo110t2s_model = Text2SemanticLightningModule(config, "ojbk", is_train=False)111t2s_model.load_state_dict(dict_s1["weight"])112if is_half == True:113 t2s_model = t2s_model.half()114t2s_model = t2s_model.to(device)115t2s_model.eval()116total = sum([param.nelement() for param in t2s_model.parameters()])117print("Number of parameter: %.2fM" % (total / 1e6))118 119 120def get_spepc(hps, filename):121 audio = load_audio(filename, int(hps.data.sampling_rate))122 audio = torch.FloatTensor(audio)123 audio_norm = audio124 audio_norm = audio_norm.unsqueeze(0)125 spec = spectrogram_torch(126 audio_norm,127 hps.data.filter_length,128 hps.data.sampling_rate,129 hps.data.hop_length,130 hps.data.win_length,131 center=False,132 )133 return spec134 135 136dict_language = {"Chinese": "zh", "English": "en", "Japanese": "ja"}137 138 139def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language):140 start_time = datetime.datetime.now()141 print(f"---START---{start_time}---")142 print(f"ref_wav_path: {ref_wav_path}")143 print(f"prompt_text: {prompt_text}")144 print(f"prompt_language: {prompt_language}")145 print(f"text: {text}")146 print(f"text_language: {text_language}")147 148 if len(prompt_text) > 100 or len(text) > 100:149 print("Input text is limited to 100 characters.")150 return "Input text is limited to 100 characters.", None151 t0 = ttime()152 prompt_text = prompt_text.strip("\n")153 prompt_language, text = prompt_language, text.strip("\n")154 with torch.no_grad():155 wav16k, _ = librosa.load(ref_wav_path, sr=16000) # 派蒙156 # length of wav16k in sec should be in 60s157 if len(wav16k) > 16000 * 60:158 print("Input audio is limited to 60 seconds.")159 return "Input audio is limited to 60 seconds.", None160 wav16k = wav16k[: int(hps.data.sampling_rate * max_sec)]161 wav16k = torch.from_numpy(wav16k)162 if is_half == True:163 wav16k = wav16k.half().to(device)164 else:165 wav16k = wav16k.to(device)166 ssl_content = ssl_model.model(wav16k.unsqueeze(0))[167 "last_hidden_state"168 ].transpose(169 1, 2170 ) # .float()171 codes = vq_model.extract_latent(ssl_content)172 prompt_semantic = codes[0, 0]173 t1 = ttime()174 prompt_language = dict_language[prompt_language]175 text_language = dict_language[text_language]176 phones1, word2ph1, norm_text1 = clean_text(prompt_text, prompt_language)177 phones1 = cleaned_text_to_sequence(phones1)178 texts = text.split("\n")179 audio_opt = []180 zero_wav = np.zeros(181 int(hps.data.sampling_rate * 0.3),182 dtype=np.float16 if is_half == True else np.float32,183 )184 for text in texts:185 phones2, word2ph2, norm_text2 = clean_text(text, text_language)186 phones2 = cleaned_text_to_sequence(phones2)187 if prompt_language == "zh":188 bert1 = get_bert_feature(norm_text1, word2ph1).to(device)189 else:190 bert1 = torch.zeros(191 (1024, len(phones1)),192 dtype=torch.float16 if is_half == True else torch.float32,193 ).to(device)194 if text_language == "zh":195 bert2 = get_bert_feature(norm_text2, word2ph2).to(device)196 else:197 bert2 = torch.zeros((1024, len(phones2))).to(bert1)198 bert = torch.cat([bert1, bert2], 1)199 200 all_phoneme_ids = torch.LongTensor(phones1 + phones2).to(device).unsqueeze(0)201 bert = bert.to(device).unsqueeze(0)202 all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device)203 prompt = prompt_semantic.unsqueeze(0).to(device)204 t2 = ttime()205 with torch.no_grad():206 # pred_semantic = t2s_model.model.infer(207 pred_semantic, idx = t2s_model.model.infer_panel(208 all_phoneme_ids,209 all_phoneme_len,210 prompt,211 bert,212 # prompt_phone_len=ph_offset,213 top_k=config["inference"]["top_k"],214 early_stop_num=hz * max_sec,215 )216 t3 = ttime()217 # print(pred_semantic.shape,idx)218 pred_semantic = pred_semantic[:, -idx:].unsqueeze(219 0220 ) # .unsqueeze(0)#mq要多unsqueeze一次221 refer = get_spepc(hps, ref_wav_path) # .to(device)222 if is_half == True:223 refer = refer.half().to(device)224 else:225 refer = refer.to(device)226 # audio = vq_model.decode(pred_semantic, all_phoneme_ids, refer).detach().cpu().numpy()[0, 0]227 audio = (228 vq_model.decode(229 pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refer230 )231 .detach()232 .cpu()233 .numpy()[0, 0]234 ) ###试试重建不带上prompt部分235 audio_opt.append(audio)236 audio_opt.append(zero_wav)237 t4 = ttime()238 end_time = datetime.datetime.now()239 dur = end_time - start_time240 print(241 f"Success! total time: {dur.seconds:.3f} sec,\ndetail time: {t1 - t0:.3f}, {t2 - t1:.3f}, {t3 - t2:.3f}, {t4 - t3:.3f}"242 )243 print(f"---END---{end_time}---")244 return (245 f"Success! total time: {dur.seconds:.3f} sec,\ndetail time: {t1 - t0:.3f}, {t2 - t1:.3f}, {t3 - t2:.3f}, {t4 - t3:.3f}",246 (247 hps.data.sampling_rate,248 (np.concatenate(audio_opt, 0) * 32768).astype(np.int16),249 ),250 )251 252 253initial_md = """254# GPT-SoVITS Zero-shot TTS Demo255 256https://github.com/RVC-Boss/GPT-SoVITS257 258*I'm not the author of this model, and I just borrowed it to make a demo.*259 260- *Input text is limited to 100 characters.*261- *Input audio is limited to 60 seconds.*262 263**License**264 265https://github.com/RVC-Boss/GPT-SoVITS/blob/main/LICENSE266 267This software is open source under the MIT License, the author does not have any control over the software, and the user is solely responsible for the use of the software and for the distribution of the sound derived from the software. 268If you do not agree with these terms and conditions, you may not use or reference any of the code or files in the package. 269"""270 271with gr.Blocks(title="GPT-SoVITS Zero-shot TTS Demo") as app:272 gr.Markdown(initial_md)273 gr.Markdown("## Upload reference audio")274 with gr.Row():275 inp_ref = gr.Audio(label="Reference audio", type="filepath")276 prompt_text = gr.Textbox(label="Transcription of reference audio")277 prompt_language = gr.Dropdown(278 label="Language of reference audio",279 choices=["Chinese", "English", "Japanese"],280 value="Japanese",281 )282 gr.Markdown("## Text to synthesize")283 with gr.Row():284 text = gr.Textbox(label="Text to synthesize")285 text_language = gr.Dropdown(286 label="Language of text",287 choices=["Chinese", "English", "Japanese"],288 value="Japanese",289 )290 inference_button = gr.Button("Synthesize", variant="primary")291 with gr.Column():292 info = gr.Textbox(label="Info")293 output = gr.Audio(label="Result")294 inference_button.click(295 get_tts_wav,296 [inp_ref, prompt_text, prompt_language, text, text_language],297 [info, output],298 )299 300app.queue(max_size=10)301app.launch(inbrowser=True)302