jhansss/SingingSDS
0
1import time2import uuid3 4import gradio as gr5import yaml6 7from characters import CHARACTERS8from pipeline import SingingDialoguePipeline9 10 11class GradioInterface:12 def __init__(self, options_config: str, default_config: str):13 self.options = self.load_config(options_config)14 self.svs_model_map = {15 model["id"]: model for model in self.options["svs_models"]16 }17 self.default_config = self.load_config(default_config)18 self.character_info = CHARACTERS19 self.current_character = self.default_config["character"]20 self.current_svs_model = (21 f"{self.default_config['language']}-{self.default_config['svs_model']}"22 )23 self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][24 self.character_info[self.current_character].default_voice25 ]26 self.pipeline = SingingDialoguePipeline(self.default_config)27 self.results = None28 29 def load_config(self, path: str):30 with open(path, "r") as f:31 return yaml.safe_load(f)32 33 def create_interface(self) -> gr.Blocks:34 try:35 with gr.Blocks(title="SingingSDS") as demo:36 gr.Markdown("# SingingSDS: Role-Playing Singing Spoken Dialogue System")37 with gr.Row():38 with gr.Column(scale=1):39 character_image = gr.Image(40 self.character_info[self.current_character].image_path,41 label="Character",42 show_label=False,43 )44 with gr.Column(scale=2):45 mic_input = gr.Audio(46 sources=["microphone", "upload"],47 type="filepath",48 label="Speak to the character",49 )50 interaction_log = gr.Textbox(51 label="Interaction Log", lines=3, interactive=False52 )53 audio_output = gr.Audio(54 label="Character's Response", type="filepath", autoplay=True55 )56 57 with gr.Row():58 metrics_button = gr.Button(59 "Evaluate Metrics", variant="secondary"60 )61 metrics_output = gr.Textbox(62 label="Evaluation Results", lines=3, interactive=False63 )64 65 gr.Markdown("## Configuration")66 with gr.Row():67 with gr.Column():68 character_radio = gr.Radio(69 label="Character Role",70 choices=list(self.character_info.keys()),71 value=self.default_config["character"],72 )73 with gr.Row():74 asr_radio = gr.Radio(75 label="ASR Model",76 choices=[77 (model["name"], model["id"])78 for model in self.options["asr_models"]79 ],80 value=self.default_config["asr_model"],81 )82 with gr.Row():83 llm_radio = gr.Radio(84 label="LLM Model",85 choices=[86 (model["name"], model["id"])87 for model in self.options["llm_models"]88 ],89 value=self.default_config["llm_model"],90 )91 with gr.Column():92 with gr.Row():93 melody_radio = gr.Radio(94 label="Melody Source",95 choices=[96 (source["name"], source["id"])97 for source in self.options["melody_sources"]98 ],99 value=self.default_config["melody_source"],100 )101 with gr.Row():102 svs_radio = gr.Radio(103 label="SVS Model",104 choices=[105 (model["name"], model["id"])106 for model in self.options["svs_models"]107 ],108 value=self.current_svs_model,109 )110 with gr.Row():111 voice_radio = gr.Radio(112 label="Singing voice",113 choices=list(114 self.svs_model_map[self.current_svs_model][115 "voices"116 ].keys()117 ),118 value=self.character_info[119 self.current_character120 ].default_voice,121 )122 character_radio.change(123 fn=self.update_character,124 inputs=character_radio,125 outputs=[character_image, voice_radio],126 )127 asr_radio.change(128 fn=self.update_asr_model, inputs=asr_radio, outputs=asr_radio129 )130 llm_radio.change(131 fn=self.update_llm_model, inputs=llm_radio, outputs=llm_radio132 )133 svs_radio.change(134 fn=self.update_svs_model,135 inputs=svs_radio,136 outputs=[svs_radio, voice_radio],137 )138 melody_radio.change(139 fn=self.update_melody_source,140 inputs=melody_radio,141 outputs=melody_radio,142 )143 voice_radio.change(144 fn=self.update_voice, inputs=voice_radio, outputs=voice_radio145 )146 mic_input.change(147 fn=self.run_pipeline,148 inputs=mic_input,149 outputs=[interaction_log, audio_output],150 )151 metrics_button.click(152 fn=self.update_metrics,153 inputs=audio_output,154 outputs=[metrics_output],155 )156 157 return demo158 except Exception as e:159 print(f"error: {e}")160 breakpoint()161 return gr.Blocks()162 163 def update_character(self, character):164 self.current_character = character165 character_voice = self.character_info[self.current_character].default_voice166 self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][167 character_voice168 ]169 return gr.update(value=self.character_info[character].image_path), gr.update(170 value=character_voice171 )172 173 def update_asr_model(self, asr_model):174 self.pipeline.set_asr_model(asr_model)175 return gr.update(value=asr_model)176 177 def update_llm_model(self, llm_model):178 self.pipeline.set_llm_model(llm_model)179 return gr.update(value=llm_model)180 181 def update_svs_model(self, svs_model):182 self.current_svs_model = svs_model183 character_voice = self.character_info[self.current_character].default_voice184 self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][185 character_voice186 ]187 self.pipeline.set_svs_model(188 self.svs_model_map[self.current_svs_model]["model_path"]189 )190 print(191 f"SVS model updated to {self.current_svs_model}. Will set gradio svs_radio to {svs_model} and voice_radio to {character_voice}"192 )193 return (194 gr.update(value=svs_model),195 gr.update(196 choices=list(197 self.svs_model_map[self.current_svs_model]["voices"].keys()198 ),199 value=character_voice,200 ),201 )202 203 def update_melody_source(self, melody_source):204 self.current_melody_source = melody_source205 return gr.update(value=self.current_melody_source)206 207 def update_voice(self, voice):208 self.current_voice = self.svs_model_map[self.current_svs_model]["voices"][voice]209 return gr.update(value=voice)210 211 def run_pipeline(self, audio_path):212 if not audio_path:213 return gr.update(value=""), gr.update(value="")214 tmp_file = f"audio_{int(time.time())}_{uuid.uuid4().hex[:8]}.wav"215 self.results = self.pipeline.run(216 audio_path,217 self.svs_model_map[self.current_svs_model]["lang"],218 self.character_info[self.current_character].prompt,219 self.current_voice,220 output_audio_path=tmp_file,221 )222 formatted_logs = f"ASR: {self.results['asr_text']}\nLLM: {self.results['llm_text']}"223 return gr.update(value=formatted_logs), gr.update(224 value=self.results["output_audio_path"]225 )226 227 def update_metrics(self, audio_path):228 if not audio_path or not self.results:229 return gr.update(value="")230 results = self.pipeline.evaluate(audio_path, **self.results)231 results.update(self.results.get("metrics", {}))232 formatted_metrics = "\n".join([f"{k}: {v}" for k, v in results.items()])233 return gr.update(value=formatted_metrics)234