sims2k/Saul-GDPR
0
1from __future__ import annotations2 3import io4import os5import re6import subprocess7import textwrap8import time9import uuid10import wave11 12import emoji13import gradio as gr14import langid15import nltk16import numpy as np17import noisereduce as nr18from huggingface_hub import HfApi19 20# Download the 'punkt' tokenizer for the NLTK library21nltk.download("punkt")22 23# will use api to restart space on a unrecoverable error24HF_TOKEN = os.environ.get("HF_TOKEN")25REPO_ID = os.environ.get("REPO_ID")26api = HfApi(token=HF_TOKEN)27 28latent_map = {}29 30def get_latents(chatbot_voice, xtts_model, voice_cleanup=False):31 global latent_map32 if chatbot_voice not in latent_map: 33 speaker_wav = f"examples/{chatbot_voice}.wav"34 if (voice_cleanup):35 try:36 cleanup_filter="lowpass=8000,highpass=75,areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02,areverse,silenceremove=start_periods=1:start_silence=0:start_threshold=0.02" 37 resample_filter="-ac 1 -ar 22050"38 out_filename = speaker_wav + str(uuid.uuid4()) + ".wav" #ffmpeg to know output format39 #we will use newer ffmpeg as that has afftn denoise filter40 shell_command = f"ffmpeg -y -i {speaker_wav} -af {cleanup_filter} {resample_filter} {out_filename}".split(" ")41 command_result = subprocess.run([item for item in shell_command], capture_output=False,text=True, check=True)42 speaker_wav=out_filename43 print("Filtered microphone input")44 except subprocess.CalledProcessError:45 # There was an error - command exited with non-zero code46 print("Error: failed filtering, use original microphone input")47 else:48 speaker_wav=speaker_wav49 # gets condition latents from the model50 # returns tuple (gpt_cond_latent, speaker_embedding)51 latent_map[chatbot_voice] = xtts_model.get_conditioning_latents(audio_path=speaker_wav)52 return latent_map[chatbot_voice]53 54 55def detect_language(prompt, xtts_supported_languages=None):56 if xtts_supported_languages is None:57 xtts_supported_languages = ["en","es","fr","de","it","pt","pl","tr","ru","nl","cs","ar","zh-cn","ja"] 58 59 # Fast language autodetection60 if len(prompt)>15:61 language_predicted=langid.classify(prompt)[0].strip() # strip need as there is space at end!62 if language_predicted == "zh": 63 #we use zh-cn on xtts64 language_predicted = "zh-cn"65 66 if language_predicted not in xtts_supported_languages:67 print(f"Detected a language not supported by xtts :{language_predicted}, switching to english for now")68 gr.Warning(f"Language detected '{language_predicted}' can not be spoken properly 'yet' ")69 language= "en"70 else:71 language = language_predicted72 print(f"Language: Predicted sentence language:{language_predicted} , using language for xtts:{language}")73 else:74 # Hard to detect language fast in short sentence, use english default75 language = "en"76 print(f"Language: Prompt is short or autodetect language disabled using english for xtts")77 78 return language79 80def get_voice_streaming(prompt, language, chatbot_voice, xtts_model, suffix="0"):81 gpt_cond_latent, speaker_embedding = get_latents(chatbot_voice, xtts_model) 82 try:83 t0 = time.time()84 chunks = xtts_model.inference_stream(85 prompt,86 language,87 gpt_cond_latent,88 speaker_embedding,89 repetition_penalty=7.0,90 temperature=0.85,91 )92 93 first_chunk = True94 for i, chunk in enumerate(chunks):95 if first_chunk:96 first_chunk_time = time.time() - t097 metrics_text = f"Latency to first audio chunk: {round(first_chunk_time*1000)} milliseconds\n"98 first_chunk = False99 #print(f"Received chunk {i} of audio length {chunk.shape[-1]}")100 101 # In case output is required to be multiple voice files102 # out_file = f'{char}_{i}.wav'103 # write(out_file, 24000, chunk.detach().cpu().numpy().squeeze())104 # audio = AudioSegment.from_file(out_file)105 # audio.export(out_file, format='wav')106 # return out_file107 # directly return chunk as bytes for streaming108 chunk = chunk.detach().cpu().numpy().squeeze()109 chunk = (chunk * 32767).astype(np.int16)110 yield chunk.tobytes()111 112 except RuntimeError as e:113 if "device-side assert" in str(e):114 # cannot do anything on cuda device side error, need tor estart115 print(116 f"Exit due to: Unrecoverable exception caused by prompt:{prompt}",117 flush=True,118 )119 gr.Warning("Unhandled Exception encounter, please retry in a minute")120 print("Cuda device-assert Runtime encountered need restart")121 122 # HF Space specific.. This error is unrecoverable need to restart space123 api.restart_space(REPO_ID=REPO_ID)124 else:125 print("RuntimeError: non device-side assert error:", str(e))126 # Does not require warning happens on empty chunk and at end127 ###gr.Warning("Unhandled Exception encounter, please retry in a minute")128 return None129 return None130 except:131 return None132 133def wave_header_chunk(frame_input=b"", channels=1, sample_width=2, sample_rate=24000):134 # This will create a wave header then append the frame input135 # It should be first on a streaming wav file136 # Other frames better should not have it (else you will hear some artifacts each chunk start)137 wav_buf = io.BytesIO()138 with wave.open(wav_buf, "wb") as vfout:139 vfout.setnchannels(channels)140 vfout.setsampwidth(sample_width)141 vfout.setframerate(sample_rate)142 vfout.writeframes(frame_input)143 144 wav_buf.seek(0)145 return wav_buf.read()146 147def format_prompt(message, history):148 system_message = f"""149 You are an empathetic, insightful, and supportive coach who helps people deal with challenges and celebrate achievements.150 You help people feel better by asking questions to reflect on and evoke feelings of positivity, gratitude, joy, and love.151 You show radical candor and tough love.152 Respond in a casual and friendly tone.153 Sprinkle in filler words, contractions, idioms, and other casual speech that we use in conversation.154 Emulate the user’s speaking style and be concise in your response.155 """156 prompt = (157 "<s>[INST]" + system_message + "[/INST]"158 )159 for user_prompt, bot_response in history:160 if user_prompt is not None:161 prompt += f"[INST] {user_prompt} [/INST]"162 prompt += f" {bot_response}</s> "163 164 if message=="":165 message="Hello"166 prompt += f"[INST] {message} [/INST]"167 return prompt168 169def generate_llm_output(170 prompt, 171 history,172 llm,173 temperature=0.8,174 max_tokens=256,175 top_p=0.95,176 stop_words=["<s>","[/INST]", "</s>"]177 ):178 temperature = float(temperature)179 if temperature < 1e-2:180 temperature = 1e-2181 top_p = float(top_p)182 183 generate_kwargs = dict(184 temperature=temperature,185 max_tokens=max_tokens,186 top_p=top_p,187 stop=stop_words188 )189 formatted_prompt = format_prompt(prompt, history)190 try:191 print("LLM Input:", formatted_prompt)192 # Local GGUF193 stream = llm(194 formatted_prompt,195 **generate_kwargs,196 stream=True,197 )198 output = ""199 for response in stream:200 character= response["choices"][0]["text"]201 202 if character in stop_words:203 # end of context204 return 205 206 if emoji.is_emoji(character):207 # Bad emoji not a meaning messes chat from next lines208 return209 210 output += response["choices"][0]["text"]211 yield output212 213 except Exception as e:214 print("Unhandled Exception: ", str(e))215 gr.Warning("Unfortunately Mistral is unable to process")216 output = "I do not know what happened but I could not understand you ."217 return output218 219def get_sentence(history, llm):220 history = [["", None]] if history is None else history 221 history[-1][1] = "" 222 sentence_list = []223 sentence_hash_list = []224 225 text_to_generate = ""226 stored_sentence = None227 stored_sentence_hash = None228 229 for character in generate_llm_output(history[-1][0], history[:-1], llm):230 history[-1][1] = character.replace("<|assistant|>","")231 # It is coming word by word232 text_to_generate = nltk.sent_tokenize(history[-1][1].replace("\n", " ").replace("<|assistant|>"," ").replace("<|ass>","").replace("[/ASST]","").replace("[/ASSI]","").replace("[/ASS]","").replace("","").strip())233 if len(text_to_generate) > 1:234 235 dif = len(text_to_generate) - len(sentence_list)236 237 if dif == 1 and len(sentence_list) != 0:238 continue239 240 if dif == 2 and len(sentence_list) != 0 and stored_sentence is not None:241 continue242 243 # All this complexity due to trying append first short sentence to next one for proper language auto-detect244 if stored_sentence is not None and stored_sentence_hash is None and dif>1:245 #means we consumed stored sentence and should look at next sentence to generate246 sentence = text_to_generate[len(sentence_list)+1]247 elif stored_sentence is not None and len(text_to_generate)>2 and stored_sentence_hash is not None:248 print("Appending stored")249 sentence = stored_sentence + text_to_generate[len(sentence_list)+1]250 stored_sentence_hash = None251 else:252 sentence = text_to_generate[len(sentence_list)]253 254 # too short sentence just append to next one if there is any255 # this is for proper language detection 256 if len(sentence)<=15 and stored_sentence_hash is None and stored_sentence is None:257 if sentence[-1] in [".","!","?"]:258 if stored_sentence_hash != hash(sentence):259 stored_sentence = sentence260 stored_sentence_hash = hash(sentence) 261 print("Storing:",stored_sentence)262 continue263 264 265 sentence_hash = hash(sentence)266 if stored_sentence_hash is not None and sentence_hash == stored_sentence_hash:267 continue268 269 if sentence_hash not in sentence_hash_list:270 sentence_hash_list.append(sentence_hash)271 sentence_list.append(sentence)272 print("New Sentence: ", sentence)273 yield (sentence, history)274 275 # return that final sentence token276 try:277 last_sentence = nltk.sent_tokenize(history[-1][1].replace("\n", " ").replace("<|ass>","").replace("[/ASST]","").replace("[/ASSI]","").replace("[/ASS]","").replace("","").strip())[-1]278 sentence_hash = hash(last_sentence)279 if sentence_hash not in sentence_hash_list:280 if stored_sentence is not None and stored_sentence_hash is not None:281 last_sentence = stored_sentence + last_sentence282 stored_sentence = stored_sentence_hash = None283 print("Last Sentence with stored:",last_sentence)284 285 sentence_hash_list.append(sentence_hash)286 sentence_list.append(last_sentence)287 print("Last Sentence: ", last_sentence)288 289 yield (last_sentence, history)290 except:291 print("ERROR on last sentence history is :", history)292 293# will generate speech audio file per sentence294def generate_speech_for_sentence(history, chatbot_voice, sentence, xtts_model, xtts_supported_languages=None, filter_output=True, return_as_byte=False):295 language = "autodetect"296 297 wav_bytestream = b""298 299 if len(sentence)==0:300 print("EMPTY SENTENCE")301 return 302 303 # Sometimes prompt </s> coming on output remove it304 # Some post process for speech only305 sentence = sentence.replace("</s>", "")306 # remove code from speech307 sentence = re.sub("```.*```", "", sentence, flags=re.DOTALL)308 sentence = re.sub("`.*`", "", sentence, flags=re.DOTALL)309 310 sentence = re.sub("\(.*\)", "", sentence, flags=re.DOTALL)311 312 sentence = sentence.replace("```", "")313 sentence = sentence.replace("...", " ")314 sentence = sentence.replace("(", " ")315 sentence = sentence.replace(")", " ")316 sentence = sentence.replace("<|assistant|>","")317 318 if len(sentence)==0:319 print("EMPTY SENTENCE after processing")320 return 321 322 # A fast fix for last chacter, may produce weird sounds if it is with text323 #if (sentence[-1] in ["!", "?", ".", ","]) or (sentence[-2] in ["!", "?", ".", ","]):324 # # just add a space325 # sentence = sentence[:-1] + " " + sentence[-1]326 327 # regex does the job well328 sentence= re.sub("([^\x00-\x7F]|\w)(\.|\。|\?|\!)",r"\1 \2\2",sentence)329 330 print("Sentence for speech:", sentence)331 332 333 try:334 SENTENCE_SPLIT_LENGTH=350335 if len(sentence)<SENTENCE_SPLIT_LENGTH:336 # no problem continue on337 sentence_list = [sentence]338 else:339 # Until now nltk likely split sentences properly but we need additional 340 # check for longer sentence and split at last possible position341 # Do whatever necessary, first break at hypens then spaces and then even split very long words342 sentence_list=textwrap.wrap(sentence,SENTENCE_SPLIT_LENGTH)343 print("SPLITTED LONG SENTENCE:",sentence_list)344 345 for sentence in sentence_list:346 347 if any(c.isalnum() for c in sentence):348 if language=="autodetect":349 #on first call autodetect, nexts sentence calls will use same language350 language = detect_language(sentence, xtts_supported_languages) 351 352 #exists at least 1 alphanumeric (utf-8) 353 audio_stream = get_voice_streaming(354 sentence, language, chatbot_voice, xtts_model355 )356 else:357 # likely got a ' or " or some other text without alphanumeric in it358 audio_stream = None 359 360 # XTTS is actually using streaming response but we are playing audio by sentence361 # If you want direct XTTS voice streaming (send each chunk to voice ) you may set DIRECT_STREAM=1 environment variable362 if audio_stream is not None:363 frame_length = 0364 for chunk in audio_stream:365 try:366 wav_bytestream += chunk367 frame_length += len(chunk)368 except:369 # hack to continue on playing. sometimes last chunk is empty , will be fixed on next TTS370 continue371 372 # Filter output for better voice373 if filter_output:374 data_s16 = np.frombuffer(wav_bytestream, dtype=np.int16, count=len(wav_bytestream)//2, offset=0)375 float_data = data_s16 * 0.5**15376 reduced_noise = nr.reduce_noise(y=float_data, sr=24000,prop_decrease =0.8,n_fft=1024)377 wav_bytestream = (reduced_noise * 32767).astype(np.int16)378 wav_bytestream = wav_bytestream.tobytes()379 380 if audio_stream is not None:381 if not return_as_byte:382 audio_unique_filename = "/tmp/"+ str(uuid.uuid4())+".wav"383 with wave.open(audio_unique_filename, "w") as f:384 f.setnchannels(1)385 # 2 bytes per sample.386 f.setsampwidth(2)387 f.setframerate(24000)388 f.writeframes(wav_bytestream)389 390 return (history , gr.update(value=audio_unique_filename, autoplay=True))391 else:392 return (history, gr.update(value=wav_bytestream, autoplay=True))393 except RuntimeError as e:394 if "device-side assert" in str(e):395 # cannot do anything on cuda device side error, need tor estart396 print(397 f"Exit due to: Unrecoverable exception caused by prompt:{sentence}",398 flush=True,399 )400 gr.Warning("Unhandled Exception encounter, please retry in a minute")401 print("Cuda device-assert Runtime encountered need restart")402 403 # HF Space specific.. This error is unrecoverable need to restart space404 api.restart_space(REPO_ID=REPO_ID)405 else:406 print("RuntimeError: non device-side assert error:", str(e))407 raise e408 409 print("All speech ended")410 return 411