Kit-Lemonfoot/vtuber_rvc_models
53
1print("Starting up. Please be patient...")
2
3import os
4import glob
5import json
6import traceback
7import logging
8import gradio as gr
9import numpy as np
10import librosa
11import torch
12import asyncio
13import edge_tts
14import yt_dlp
15import ffmpeg
16import subprocess
17import sys
18import io
19import wave
20from datetime import datetime
21from fairseq import checkpoint_utils
22from lib.infer_pack.models import (
23 SynthesizerTrnMs256NSFsid,
24 SynthesizerTrnMs256NSFsid_nono,
25 SynthesizerTrnMs768NSFsid,
26 SynthesizerTrnMs768NSFsid_nono,
27)
28from vc_infer_pipeline import VC
29from config import Config
30from edgetts_db import tts_order_voice
31
32#fuck intel
33os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
34
35config = Config()
36logging.getLogger("numba").setLevel(logging.WARNING)
37limitation = os.getenv("SYSTEM") == "spaces"
38#limitation=True
39language_dict = tts_order_voice
40
41authors = ["dacoolkid44", "Hijack", "Maki Ligon", "megaaziib", "Kit Lemonfoot", "yeey5", "Sui", "MahdeenSky"]
42
43f0method_mode = []
44if limitation is True:
45 f0method_info = "PM is better for testing, RMVPE is better for finalized generations. (Default: PM)"
46 f0method_mode = ["pm", "rmvpe"]
47else:
48 f0method_info = "PM is fast but low quality, crepe and harvest are slow but good quality, RMVPE is the best of both worlds. (Default: PM)"
49 f0method_mode = ["pm", "crepe", "harvest", "rmvpe"]
50
51#Eagerload VCs
52print("Preloading VCs...")
53vcArr=[]
54vcArr.append(VC(32000, config))
55vcArr.append(VC(40000, config))
56vcArr.append(VC(48000, config))
57
58def infer(name, path, index, vc_input, vc_upload, tts_text, tts_voice, f0_up_key, f0_method, index_rate, filter_radius, resample_sr, rms_mix_rate, protect):
59 try:
60 #Setup audio
61 audio=None
62 #Determine audio mode
63 #TTS takes priority over uploads.
64 #Uploads takes priority over paths.
65 vc_audio_mode = ""
66 #Edge-TTS
67 if(tts_text):
68 vc_audio_mode = "ETTS"
69 if len(tts_text) > 250 and limitation:
70 return "Text is too long.", None
71 if tts_text is None or tts_voice is None or tts_text=="":
72 return "You need to enter text and select a voice.", None
73 voice = language_dict[tts_voice]
74 try:
75 asyncio.run(edge_tts.Communicate(tts_text, voice).save("tts.mp3"))
76 except:
77 print("Failed to get E-TTS handle. A restart may be needed soon.")
78 return "ERROR: Failed to communicate with Edge-TTS. The Edge-TTS service may be down or cannot communicate. Please try another method or try again later.", None
79 try:
80 audio, sr = librosa.load("tts.mp3", sr=16000, mono=True)
81 except:
82 return "ERROR: Invalid characters for the chosen TTS speaker. (Change your TTS speaker to one that supports your language!)", None
83 duration = audio.shape[0] / sr
84 if duration > 30 and limitation:
85 return "Your text generated an audio that was too long.", None
86 vc_input = "tts.mp3"
87 #File upload
88 elif(vc_upload):
89 vc_audio_mode = "Upload"
90 sampling_rate, audio = vc_upload
91 duration = audio.shape[0] / sampling_rate
92 if duration > 60 and limitation:
93 return "Too long! Please upload an audio file that is less than 1 minute.", None
94 audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
95 if len(audio.shape) > 1:
96 audio = librosa.to_mono(audio.transpose(1, 0))
97 if sampling_rate != 16000:
98 audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
99 tts_text = "Uploaded Audio"
100 #YouTube or path
101 elif(vc_input):
102 audio, sr = librosa.load(vc_input, sr=16000, mono=True)
103 vc_audio_mode = "YouTube"
104 tts_text = "YouTube Audio"
105 else:
106 return "Please upload or choose some type of audio.", None
107
108 if audio is None:
109 if vc_audio_mode == "ETTS":
110 print("Failed to get E-TTS handle. A restart may be needed soon.")
111 return "ERROR: Failed to obtain a correct response from Edge-TTS. The Edge-TTS service may be down or unable to communicate. Please try another method or try again later.", None
112 return "ERROR: Unknown audio error. Please try again.", None
113
114 times = [0, 0, 0]
115 f0_up_key = int(f0_up_key)
116
117 #Setup model
118 cpt = torch.load(f"{path}", map_location="cpu")
119 tgt_sr = cpt["config"][-1]
120 cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] # n_spk
121 if_f0 = cpt.get("f0", 1)
122 version = cpt.get("version", "v1")
123 if version == "v1":
124 if if_f0 == 1:
125 net_g = SynthesizerTrnMs256NSFsid(*cpt["config"], is_half=config.is_half)
126 else:
127 net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"])
128 model_version = "V1"
129 elif version == "v2":
130 if if_f0 == 1:
131 net_g = SynthesizerTrnMs768NSFsid(*cpt["config"], is_half=config.is_half)
132 else:
133 net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"])
134 model_version = "V2"
135 del net_g.enc_q
136 print(net_g.load_state_dict(cpt["weight"], strict=False))
137 net_g.eval().to(config.device)
138 if config.is_half:
139 net_g = net_g.half()
140 else:
141 net_g = net_g.float()
142 vcIdx = int((tgt_sr/8000)-4)
143
144 #Gen audio
145 audio_opt = vcArr[vcIdx].pipeline(
146 hubert_model,
147 net_g,
148 0,
149 audio,
150 vc_input,
151 times,
152 f0_up_key,
153 f0_method,
154 index,
155 # file_big_npy,
156 index_rate,
157 if_f0,
158 filter_radius,
159 tgt_sr,
160 resample_sr,
161 rms_mix_rate,
162 version,
163 protect,
164 f0_file=None,
165 )
166 info = f"[{datetime.now().strftime('%Y-%m-%d %H:%M')}]: npy: {times[0]}, f0: {times[1]}s, infer: {times[2]}s"
167 print(f"Successful inference with model {name} | {tts_text} | {info}")
168 del net_g, cpt
169 return info, (tgt_sr, audio_opt)
170 except:
171 info = traceback.format_exc()
172 print(info)
173 return info, (None, None)
174
175def load_model():
176 categories = []
177 with open("weights/folder_info.json", "r", encoding="utf-8") as f:
178 folder_info = json.load(f)
179 for category_name, category_info in folder_info.items():
180 if not category_info['enable']:
181 continue
182 category_title = category_info['title']
183 category_folder = category_info['folder_path']
184 models = []
185 print(f"Creating category {category_title}...")
186 with open(f"weights/{category_folder}/model_info.json", "r", encoding="utf-8") as f:
187 models_info = json.load(f)
188 for character_name, info in models_info.items():
189 if not info['enable']:
190 continue
191 model_title = info['title']
192 model_name = info['model_path']
193 model_author = info.get("author", None)
194 model_cover = f"weights/{category_folder}/{character_name}/{info['cover']}"
195 model_index = f"weights/{category_folder}/{character_name}/{info['feature_retrieval_library']}"
196 if info['feature_retrieval_library'] == "None":
197 model_index = None
198 if model_index:
199 assert os.path.exists(model_index), f"Model {model_title} failed to load index."
200 if not (model_author in authors or "/" in model_author or "&" in model_author):
201 authors.append(model_author)
202 model_path = f"weights/{category_folder}/{character_name}/{model_name}"
203 cpt = torch.load(f"weights/{category_folder}/{character_name}/{model_name}", map_location="cpu")
204 model_version = cpt.get("version", "v1")
205 print(f"Indexed model {model_title} by {model_author} ({model_version})")
206 models.append((character_name, model_title, model_author, model_cover, model_version, model_path, model_index))
207 del cpt
208 categories.append([category_title, category_folder, models])
209 return categories
210
211def cut_vocal_and_inst(url, audio_provider, split_model):
212 if url != "":
213 if not os.path.exists("dl_audio"):
214 os.mkdir("dl_audio")
215 if audio_provider == "Youtube":
216 ydl_opts = {
217 'format': 'bestaudio/best',
218 'postprocessors': [{
219 'key': 'FFmpegExtractAudio',
220 'preferredcodec': 'wav',
221 }],
222 "outtmpl": 'dl_audio/youtube_audio',
223 }
224 with yt_dlp.YoutubeDL(ydl_opts) as ydl:
225 ydl.download([url])
226 audio_path = "dl_audio/youtube_audio.wav"
227 else:
228 # Spotify doesnt work.
229 # Need to find other solution soon.
230 '''
231 command = f"spotdl download {url} --output dl_audio/.wav"
232 result = subprocess.run(command.split(), stdout=subprocess.PIPE)
233 print(result.stdout.decode())
234 audio_path = "dl_audio/spotify_audio.wav"
235 '''
236 if split_model == "htdemucs":
237 command = f"demucs --two-stems=vocals {audio_path} -o output"
238 result = subprocess.run(command.split(), stdout=subprocess.PIPE)
239 print(result.stdout.decode())
240 return "output/htdemucs/youtube_audio/vocals.wav", "output/htdemucs/youtube_audio/no_vocals.wav", audio_path, "output/htdemucs/youtube_audio/vocals.wav"
241 else:
242 command = f"demucs --two-stems=vocals -n mdx_extra_q {audio_path} -o output"
243 result = subprocess.run(command.split(), stdout=subprocess.PIPE)
244 print(result.stdout.decode())
245 return "output/mdx_extra_q/youtube_audio/vocals.wav", "output/mdx_extra_q/youtube_audio/no_vocals.wav", audio_path, "output/mdx_extra_q/youtube_audio/vocals.wav"
246 else:
247 raise gr.Error("URL Required!")
248 return None, None, None, None
249
250def load_hubert():
251 global hubert_model
252 models, _, _ = checkpoint_utils.load_model_ensemble_and_task(
253 ["hubert_base.pt"],
254 suffix="",
255 )
256 hubert_model = models[0]
257 hubert_model = hubert_model.to(config.device)
258 if config.is_half:
259 hubert_model = hubert_model.half()
260 else:
261 hubert_model = hubert_model.float()
262 hubert_model.eval()
263
264if __name__ == '__main__':
265 load_hubert()
266 categories = load_model()
267 voices = list(language_dict.keys())
268
269 # Gradio preloading
270 # Input and Upload
271 vc_upload = gr.Audio(label="Upload or record an audio file", interactive=True)
272 # Youtube
273 vc_input = gr.Textbox(label="Input audio path", visible=False)
274 vc_download_audio = gr.Dropdown(label="Provider", choices=["Youtube"], allow_custom_value=False, value="Youtube", info="Select provider (Default: Youtube)")
275 vc_link = gr.Textbox(label="Youtube URL", info="Example: https://www.youtube.com/watch?v=Nc0sB1Bmf-A", placeholder="https://www.youtube.com/watch?v=...")
276 vc_split_model = gr.Dropdown(label="Splitter Model", choices=["htdemucs", "mdx_extra_q"], allow_custom_value=False, value="htdemucs", info="Select the splitter model (Default: htdemucs)")
277 vc_split = gr.Button("Split Audio", variant="primary")
278 vc_vocal_preview = gr.Audio(label="Vocal Preview")
279 vc_inst_preview = gr.Audio(label="Instrumental Preview")
280 vc_audio_preview = gr.Audio(label="Audio Preview")
281 # TTS
282 tts_text = gr.Textbox(label="TTS text", info="Text to speech input (There is a limit of 250 characters)", interactive=True)
283 tts_voice = gr.Dropdown(label="Edge-TTS speaker", choices=voices, allow_custom_value=False, value="English-Ana (Female)", interactive=True)
284 # Other settings
285 vc_transform0 = gr.Number(label="Transpose", value=0, info='Type "12" to change from male to female voice. Type "-12" to change female to male voice')
286 f0method0 = gr.Radio(
287 label="Pitch extraction algorithm",
288 info=f0method_info,
289 choices=f0method_mode,
290 value="pm",
291 interactive=True
292 )
293 index_rate1 = gr.Slider(
294 minimum=0,
295 maximum=1,
296 label="Retrieval feature ratio",
297 info="Accent control. Too high will usually sound too robotic. (Default: 0.4)",
298 value=0.4,
299 interactive=True,
300 )
301 filter_radius0 = gr.Slider(
302 minimum=0,
303 maximum=7,
304 label="Apply Median Filtering",
305 info="The value represents the filter radius and can reduce breathiness.",
306 value=1,
307 step=1,
308 interactive=True,
309 )
310 resample_sr0 = gr.Slider(
311 minimum=0,
312 maximum=48000,
313 label="Resample the output audio",
314 info="Resample the output audio in post-processing to the final sample rate. Set to 0 for no resampling.",
315 value=0,
316 step=1,
317 interactive=True,
318 )
319 rms_mix_rate0 = gr.Slider(
320 minimum=0,
321 maximum=1,
322 label="Volume Envelope",
323 info="Use the volume envelope of the input to replace or mix with the volume envelope of the output. The closer the ratio is to 1, the more the output envelope is used",
324 value=1,
325 interactive=True,
326 )
327 protect0 = gr.Slider(
328 minimum=0,
329 maximum=0.5,
330 label="Voice Protection",
331 info="Protect voiceless consonants and breath sounds to prevent artifacts such as tearing in electronic music. Set to 0.5 to disable. Decrease the value to increase protection, but it may reduce indexing accuracy",
332 value=0.23,
333 step=0.01,
334 interactive=True,
335 )
336
337 with gr.Blocks(theme=gr.themes.Base()) as app:
338 gr.Markdown(
339 "# <center> VTuber RVC Models\n"
340 "### <center> Space by Kit Lemonfoot / Noel Shirogane's High Flying Birds"
341 "<center> Original space by megaaziib & zomehwh\n"
342 "### <center> Please credit the original model authors if you use this Space."
343 "<center>Do no evil.\n\n"
344 "[](https://colab.research.google.com/drive/1Til3SY7-X0x3Wss3YXlgfq8go39DzWHk)\n\n"
345 )
346 gr.Markdown("<center> Looking for more models? <a href=\"https://docs.google.com/spreadsheets/d/1tvZSggOsZGAPjbMrWOAAaoJJFpJuQlwUEQCf5x1ssO8\">Check out the VTuber AI Model Tracking spreadsheet!</a>")
347 for (folder_title, folder, models) in categories:
348 with gr.TabItem(folder_title):
349 with gr.Tabs():
350 if not models:
351 gr.Markdown("# <center> No Model Loaded.")
352 gr.Markdown("## <center> Please add model or fix your model path.")
353 continue
354 for (name, title, author, cover, model_version, model_path, model_index) in models:
355 with gr.TabItem(name):
356 with gr.Row():
357 with gr.Column():
358 gr.Markdown(
359 '<div align="center">'
360 f'<div>{title}</div>\n'+
361 f'<div>RVC {model_version} Model</div>\n'+
362 (f'<div>Model author: {author}</div>' if author else "")+
363 (f'<img style="width:auto;height:300px;" src="file/{cover}"></img>' if cover else "")+
364 '</div>'
365 )
366 with gr.Column():
367 vc_log = gr.Textbox(label="Output Information", interactive=False)
368 vc_output = gr.Audio(label="Output Audio", interactive=False)
369 #This is a fucking stupid solution but Gradio refuses to pass in values unless I do this.
370 vc_name = gr.Textbox(value=title, visible=False, interactive=False)
371 vc_mp = gr.Textbox(value=model_path, visible=False, interactive=False)
372 vc_mi = gr.Textbox(value=model_index, visible=False, interactive=False)
373 vc_convert = gr.Button("Convert", variant="primary")
374
375 vc_convert.click(
376 fn=infer,
377 inputs=[
378 vc_name,
379 vc_mp,
380 vc_mi,
381 vc_input,
382 vc_upload,
383 tts_text,
384 tts_voice,
385 vc_transform0,
386 f0method0,
387 index_rate1,
388 filter_radius0,
389 resample_sr0,
390 rms_mix_rate0,
391 protect0
392 ],
393 outputs=[vc_log, vc_output]
394 )
395
396 with gr.Row():
397 with gr.Column():
398 with gr.Tab("Edge-TTS"):
399 tts_text.render()
400 tts_voice.render()
401 with gr.Tab("Upload/Record"):
402 vc_input.render()
403 vc_upload.render()
404 if(not limitation):
405 with gr.Tab("YouTube"):
406 vc_download_audio.render()
407 vc_link.render()
408 vc_split_model.render()
409 vc_split.render()
410 vc_vocal_preview.render()
411 vc_inst_preview.render()
412 vc_audio_preview.render()
413 with gr.Column():
414 vc_transform0.render()
415 f0method0.render()
416 index_rate1.render()
417 with gr.Accordion("Advanced Options", open=False):
418 filter_radius0.render()
419 resample_sr0.render()
420 rms_mix_rate0.render()
421 protect0.render()
422
423 vc_split.click(
424 fn=cut_vocal_and_inst,
425 inputs=[vc_link, vc_download_audio, vc_split_model],
426 outputs=[vc_vocal_preview, vc_inst_preview, vc_audio_preview, vc_input]
427 )
428
429 authStr=", ".join(authors)
430 gr.Markdown(
431 "## <center>Credit to:\n"
432 "#### <center>Original devs:\n"
433 "<center>the RVC Project, lj1995, zomehwh, sysf\n\n"
434 "#### <center>Model creators:\n"
435 f"<center>{authStr}\n"
436 )
437
438if limitation is True:
439 app.queue(max_size=20, api_open=config.api).launch(allowed_paths=["/"])
440else:
441 app.queue(max_size=20, api_open=config.api).launch(allowed_paths=["/"], share=False)
442