undetectable/voice-clone
1
1import sys2from TTS.api import TTS3import io, os, stat4import subprocess5import random6from zipfile import ZipFile7import uuid8import time9import torch10import torchaudio11 12 13#download for mecab14# os.system('python -m unidic download')15 16# By using XTTS you agree to CPML license https://coqui.ai/cpml17os.environ["COQUI_TOS_AGREED"] = "1"18 19# langid is used to detect language for longer text20# Most users expect text to be their own language, there is checkbox to disable it21import langid22import base6423import csv24from io import StringIO25import datetime26import re27 28import gradio as gr29from scipy.io.wavfile import write30from pydub import AudioSegment31 32from TTS.api import TTS33from TTS.tts.configs.xtts_config import XttsConfig34from TTS.tts.models.xtts import Xtts35from TTS.utils.generic_utils import get_user_data_dir36 37HF_TOKEN = os.environ.get("HF_TOKEN")38 39from huggingface_hub import HfApi40 41# will use api to restart space on a unrecoverable error42api = HfApi(token=HF_TOKEN)43repo_id = "coqui/xtts"44 45# Use never ffmpeg binary for Ubuntu20 to use denoising for microphone input46print("Export newer ffmpeg binary for denoise filter")47ZipFile("ffmpeg.zip").extractall()48print("Make ffmpeg binary executable")49st = os.stat("ffmpeg")50os.chmod("ffmpeg", st.st_mode | stat.S_IEXEC)51 52# This will trigger downloading model53print("Downloading if not downloaded Coqui XTTS V2")54from TTS.utils.manage import ModelManager55 56model_name = "tts_models/multilingual/multi-dataset/xtts_v2"57ModelManager().download_model(model_name)58model_path = os.path.join(get_user_data_dir("tts"), model_name.replace("/", "--"))59print("XTTS downloaded")60 61config = XttsConfig()62config.load_json(os.path.join(model_path, "config.json"))63 64model = Xtts.init_from_config(config)65model.load_checkpoint(66 config,67 checkpoint_path=os.path.join(model_path, "model.pth"),68 vocab_path=os.path.join(model_path, "vocab.json"),69 eval=True,70 use_deepspeed=True,71)72model.cuda()73 74# This is for debugging purposes only75DEVICE_ASSERT_DETECTED = 076DEVICE_ASSERT_PROMPT = None77DEVICE_ASSERT_LANG = None78 79supported_languages = config.languages80 81def predict(82 prompt,83 language,84 audio_file_pth,85 mic_file_path,86 use_mic,87 voice_cleanup,88 no_lang_auto_detect,89 agree,90):91 if agree == True:92 if language not in supported_languages:93 gr.Warning(94 f"Language you put {language} in is not in is not in our Supported Languages, please choose from dropdown"95 )96 97 return (98 None,99 None,100 None,101 None,102 )103 104 language_predicted = langid.classify(prompt)[105 0106 ].strip() # strip need as there is space at end!107 108 # tts expects chinese as zh-cn109 if language_predicted == "zh":110 # we use zh-cn111 language_predicted = "zh-cn"112 113 print(f"Detected language:{language_predicted}, Chosen language:{language}")114 115 # After text character length 15 trigger language detection116 if len(prompt) > 15:117 # allow any language for short text as some may be common118 # If user unchecks language autodetection it will not trigger119 # You may remove this completely for own use120 if language_predicted != language and not no_lang_auto_detect:121 # Please duplicate and remove this check if you really want this122 # Or auto-detector fails to identify language (which it can on pretty short text or mixed text)123 gr.Warning(124 f"It looks like your text isn’t the language you chose , if you’re sure the text is the same language you chose, please check disable language auto-detection checkbox"125 )126 127 return (128 None,129 None,130 None,131 None,132 )133 134 if use_mic == True:135 if mic_file_path is not None:136 speaker_wav = mic_file_path137 else:138 gr.Warning(139 "Please record your voice with Microphone, or uncheck Use Microphone to use reference audios"140 )141 return (142 None,143 None,144 None,145 None,146 )147 148 else:149 speaker_wav = audio_file_pth150 151 # Filtering for microphone input, as it has BG noise, maybe silence in beginning and end152 # This is fast filtering not perfect153 154 # Apply all on demand155 lowpassfilter = denoise = trim = loudness = True156 157 if lowpassfilter:158 lowpass_highpass = "lowpass=8000,highpass=75,"159 else:160 lowpass_highpass = ""161 162 if trim:163 # better to remove silence in beginning and end for microphone164 trim_silence = "areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,"165 else:166 trim_silence = ""167 168 if voice_cleanup:169 try:170 out_filename = (171 speaker_wav + str(uuid.uuid4()) + ".wav"172 ) # ffmpeg to know output format173 174 # we will use newer ffmpeg as that has afftn denoise filter175 shell_command = f"./ffmpeg -y -i {speaker_wav} -af {lowpass_highpass}{trim_silence} {out_filename}".split(176 " "177 )178 179 command_result = subprocess.run(180 [item for item in shell_command],181 capture_output=False,182 text=True,183 check=True,184 )185 speaker_wav = out_filename186 print("Filtered microphone input")187 except subprocess.CalledProcessError:188 # There was an error - command exited with non-zero code189 print("Error: failed filtering, use original microphone input")190 else:191 speaker_wav = speaker_wav192 193 if len(prompt) < 2:194 gr.Warning("Please give a longer prompt text")195 return (196 None,197 None,198 None,199 None,200 )201 if len(prompt) > 200:202 gr.Warning(203 "Text length limited to 200 characters for this demo, please try shorter text. You can clone this space and edit code for your own usage"204 )205 return (206 None,207 None,208 None,209 None,210 )211 global DEVICE_ASSERT_DETECTED212 if DEVICE_ASSERT_DETECTED:213 global DEVICE_ASSERT_PROMPT214 global DEVICE_ASSERT_LANG215 # It will likely never come here as we restart space on first unrecoverable error now216 print(217 f"Unrecoverable exception caused by language:{DEVICE_ASSERT_LANG} prompt:{DEVICE_ASSERT_PROMPT}"218 )219 220 # HF Space specific.. This error is unrecoverable need to restart space221 space = api.get_space_runtime(repo_id=repo_id)222 if space.stage!="BUILDING":223 api.restart_space(repo_id=repo_id)224 else:225 print("TRIED TO RESTART but space is building")226 227 try:228 metrics_text = ""229 t_latent = time.time()230 231 # note diffusion_conditioning not used on hifigan (default mode), it will be empty but need to pass it to model.inference232 try:233 (234 gpt_cond_latent,235 speaker_embedding,236 ) = model.get_conditioning_latents(audio_path=speaker_wav, gpt_cond_len=30, gpt_cond_chunk_len=4, max_ref_length=60)237 except Exception as e:238 print("Speaker encoding error", str(e))239 gr.Warning(240 "It appears something wrong with reference, did you unmute your microphone?"241 )242 return (243 None,244 None,245 None,246 None,247 )248 249 latent_calculation_time = time.time() - t_latent250 # metrics_text=f"Embedding calculation time: {latent_calculation_time:.2f} seconds\n"251 252 # temporary comma fix253 prompt= re.sub("([^\x00-\x7F]|\w)(\.|\。|\?)",r"\1 \2\2",prompt)254 255 wav_chunks = []256 ## Direct mode257 258 print("I: Generating new audio...")259 t0 = time.time()260 out = model.inference(261 prompt,262 language,263 gpt_cond_latent,264 speaker_embedding,265 repetition_penalty=5.0,266 temperature=0.75,267 )268 inference_time = time.time() - t0269 print(f"I: Time to generate audio: {round(inference_time*1000)} milliseconds")270 metrics_text+=f"Time to generate audio: {round(inference_time*1000)} milliseconds\n"271 real_time_factor= (time.time() - t0) / out['wav'].shape[-1] * 24000272 print(f"Real-time factor (RTF): {real_time_factor}")273 metrics_text+=f"Real-time factor (RTF): {real_time_factor:.2f}\n"274 torchaudio.save("output.wav", torch.tensor(out["wav"]).unsqueeze(0), 24000)275 276 277 """278 print("I: Generating new audio in streaming mode...")279 t0 = time.time()280 chunks = model.inference_stream(281 prompt,282 language,283 gpt_cond_latent,284 speaker_embedding,285 repetition_penalty=7.0,286 temperature=0.85,287 )288 289 first_chunk = True290 for i, chunk in enumerate(chunks):291 if first_chunk:292 first_chunk_time = time.time() - t0293 metrics_text += f"Latency to first audio chunk: {round(first_chunk_time*1000)} milliseconds\n"294 first_chunk = False295 wav_chunks.append(chunk)296 print(f"Received chunk {i} of audio length {chunk.shape[-1]}")297 inference_time = time.time() - t0298 print(299 f"I: Time to generate audio: {round(inference_time*1000)} milliseconds"300 )301 #metrics_text += (302 # f"Time to generate audio: {round(inference_time*1000)} milliseconds\n"303 #)304 305 wav = torch.cat(wav_chunks, dim=0)306 print(wav.shape)307 real_time_factor = (time.time() - t0) / wav.shape[0] * 24000308 print(f"Real-time factor (RTF): {real_time_factor}")309 metrics_text += f"Real-time factor (RTF): {real_time_factor:.2f}\n"310 311 torchaudio.save("output.wav", wav.squeeze().unsqueeze(0).cpu(), 24000)312 """313 314 except RuntimeError as e:315 if "device-side assert" in str(e):316 # cannot do anything on cuda device side error, need tor estart317 print(318 f"Exit due to: Unrecoverable exception caused by language:{language} prompt:{prompt}",319 flush=True,320 )321 gr.Warning("Unhandled Exception encounter, please retry in a minute")322 print("Cuda device-assert Runtime encountered need restart")323 if not DEVICE_ASSERT_DETECTED:324 DEVICE_ASSERT_DETECTED = 1325 DEVICE_ASSERT_PROMPT = prompt326 DEVICE_ASSERT_LANG = language327 328 # just before restarting save what caused the issue so we can handle it in future329 # Uploading Error data only happens for unrecovarable error330 error_time = datetime.datetime.now().strftime("%d-%m-%Y-%H:%M:%S")331 error_data = [332 error_time,333 prompt,334 language,335 audio_file_pth,336 mic_file_path,337 use_mic,338 voice_cleanup,339 no_lang_auto_detect,340 agree,341 ]342 error_data = [str(e) if type(e) != str else e for e in error_data]343 print(error_data)344 print(speaker_wav)345 write_io = StringIO()346 csv.writer(write_io).writerows([error_data])347 csv_upload = write_io.getvalue().encode()348 349 filename = error_time + "_" + str(uuid.uuid4()) + ".csv"350 print("Writing error csv")351 error_api = HfApi()352 error_api.upload_file(353 path_or_fileobj=csv_upload,354 path_in_repo=filename,355 repo_id="coqui/xtts-flagged-dataset",356 repo_type="dataset",357 )358 359 # speaker_wav360 print("Writing error reference audio")361 speaker_filename = (362 error_time + "_reference_" + str(uuid.uuid4()) + ".wav"363 )364 error_api = HfApi()365 error_api.upload_file(366 path_or_fileobj=speaker_wav,367 path_in_repo=speaker_filename,368 repo_id="coqui/xtts-flagged-dataset",369 repo_type="dataset",370 )371 372 # HF Space specific.. This error is unrecoverable need to restart space373 space = api.get_space_runtime(repo_id=repo_id)374 if space.stage!="BUILDING":375 api.restart_space(repo_id=repo_id)376 else:377 print("TRIED TO RESTART but space is building")378 379 else:380 if "Failed to decode" in str(e):381 print("Speaker encoding error", str(e))382 gr.Warning(383 "It appears something wrong with reference, did you unmute your microphone?"384 )385 else:386 print("RuntimeError: non device-side assert error:", str(e))387 gr.Warning("Something unexpected happened please retry again.")388 return (389 None,390 None,391 None,392 None,393 )394 return (395 gr.make_waveform(396 audio="output.wav",397 ),398 "output.wav",399 metrics_text,400 speaker_wav,401 )402 else:403 gr.Warning("Please accept the Terms & Condition!")404 return (405 None,406 None,407 None,408 None,409 )410 411 412 413with gr.Blocks(analytics_enabled=False) as demo:414 with gr.Row():415 with gr.Column():416 input_text_gr = gr.Textbox(417 label="Text Prompt",418 info="One or two sentences at a time is better. Up to 200 text characters.",419 value="Hi there, I'm your new voice clone. Try your best to upload quality audio.",420 )421 language_gr = gr.Dropdown(422 label="Language",423 info="Select an output language for the synthesised speech",424 choices=[425 "en",426 "es",427 "fr",428 "de",429 "it",430 "pt",431 "pl",432 "tr",433 "ru",434 "nl",435 "cs",436 "ar",437 "zh-cn",438 "ja",439 "ko",440 "hu",441 "hi"442 ],443 max_choices=1,444 value="en",445 )446 ref_gr = gr.Audio(447 label="Reference Audio",448 info="Click on the ✎ button to upload your own target speaker audio",449 type="filepath",450 value="examples/female.wav",451 )452 mic_gr = gr.Audio(453 source="microphone",454 type="filepath",455 info="Use your microphone to record audio",456 label="Use Microphone for Reference",457 )458 use_mic_gr = gr.Checkbox(459 label="Use Microphone",460 value=False,461 info="Notice: Microphone input may not work properly under traffic",462 )463 clean_ref_gr = gr.Checkbox(464 label="Cleanup Reference Voice",465 value=False,466 info="This check can improve output if your microphone or reference voice is noisy",467 )468 auto_det_lang_gr = gr.Checkbox(469 label="Do not use language auto-detect",470 value=False,471 info="Check to disable language auto-detection",472 )473 tos_gr = gr.Checkbox(474 label="Agree",475 value=False,476 info="I agree to the terms of the CPML: https://coqui.ai/cpml",477 )478 479 tts_button = gr.Button("Send", elem_id="send-btn", visible=True)480 481 482 with gr.Column():483 video_gr = gr.Video(label="Waveform Visual")484 audio_gr = gr.Audio(label="Synthesised Audio", autoplay=True)485 out_text_gr = gr.Text(label="Metrics")486 ref_audio_gr = gr.Audio(label="Reference Audio Used")487 488 with gr.Row():489 gr.Examples(examples,490 label="Examples",491 inputs=[input_text_gr, language_gr, ref_gr, mic_gr, use_mic_gr, clean_ref_gr, auto_det_lang_gr, tos_gr],492 outputs=[video_gr, audio_gr, out_text_gr, ref_audio_gr],493 fn=predict,494 cache_examples=False,)495 496 tts_button.click(predict, [input_text_gr, language_gr, ref_gr, mic_gr, use_mic_gr, clean_ref_gr, auto_det_lang_gr, tos_gr], outputs=[video_gr, audio_gr, out_text_gr, ref_audio_gr])497 498demo.queue() 499demo.launch(debug=True, show_api=True)