Surunrun/SoulX-Singer

1import random2import sys3import traceback4import gc5from datetime import datetime6from pathlib import Path7from typing import Literal8 9import gradio as gr10import librosa11import numpy as np12import soundfile as sf13import torch14 15import spaces16from preprocess.pipeline import PreprocessPipeline17from soulxsinger.utils.file_utils import load_config18from cli.inference_svc import build_model as build_svc_model, process as svc_process19 20 21ROOT = Path(__file__).parent22SAMPLE_RATE = 4410023PROMPT_MAX_SEC_DEFAULT = 3024TARGET_MAX_SEC_DEFAULT = 60025 26# Example rows: only [prompt_audio, target_audio]; other params use UI defaults when running27EXAMPLE_LIST = [28    [str(ROOT / "example/audio/zh_prompt.mp3"), str(ROOT / "example/audio/zh_target.mp3")],29    [str(ROOT / "example/audio/en_prompt.mp3"), str(ROOT / "example/audio/en_target.mp3")],30    [str(ROOT / "example/audio/svc_webui/Sun Yanzi.mp3"), str(ROOT / "example/audio/svc_webui/I'm Yours.mp3")],31    [str(ROOT / "example/audio/svc_webui/Sun Yanzi.mp3"), str(ROOT / "example/audio/svc_webui/传奇.mp3")],32    [str(ROOT / "example/audio/svc_webui/Sun Yanzi.mp3"), str(ROOT / "example/audio/svc_webui/君が好きだと叫びたい.mp3")],33    [str(ROOT / "example/audio/svc_webui/Sun Yanzi.mp3"), str(ROOT / "example/audio/svc_webui/富士山下.mp3")],34]35 36_I18N = dict(37	display_lang_label=dict(en="Display Language", zh="显示语言"),38	title=dict(en="## SoulX-Singer SVC", zh="## SoulX-Singer SVC"),39	prompt_audio_label=dict(en=f"Prompt audio", zh=f"Prompt 音频"),40	target_audio_label=dict(en=f"Target audio", zh=f"Target 音频"),41	prompt_vocal_sep_label=dict(en="Prompt vocal separation", zh="Prompt 人声分离"),42	target_vocal_sep_label=dict(en="Target vocal separation", zh="Target 人声分离"),43	auto_shift_label=dict(en="Auto pitch shift", zh="自动变调"),44	auto_mix_acc_label=dict(en="Auto mix accompaniment", zh="自动混合伴奏"),45	pitch_shift_label=dict(en="Pitch shift (semitones)", zh="指定变调（半音）"),46	n_step_label=dict(en="diffusion steps", zh="采样步数"),47	cfg_label=dict(en="cfg scale", zh="cfg系数"),48	seed_label=dict(en="Seed", zh="种子"),49	examples_label=dict(en="Examples", zh="示例"),50	run_btn=dict(en="🎤Singing Voice Conversion", zh="🎤歌声转换"),51	output_audio_label=dict(en="Generated audio", zh="合成结果音频"),52	warn_missing_audio=dict(en="Please provide both prompt audio and target audio.", zh="请同时上传 Prompt 与 Target 音频。"),53	instruction_title=dict(en="Usage", zh="使用说明"),54	instruction_p1=dict(55        en="Upload the Prompt and Target audio, and configure the parameters",56        zh="上传 Prompt 与 Target 音频，并配置相关参数",57    ),58    instruction_p2=dict(59        en="Click「🎤Singing Voice Conversion」to start singing voice conversion.",60        zh="点击「🎤歌声转换」开始最终生成。",61    ),62	tips_title=dict(en="Tips", zh="提示"),63	tip_p1=dict(64        en="Input: The Prompt audio is recommended to be a clean and clear singing voice, while the Target audio can be either a pure vocal or a mixture with accompaniment. If the audio contains accompaniment, please check the vocal separation option.",65        zh="输入：Prompt 音频建议是干净清晰的歌声，Target 音频可以是纯歌声或伴奏，这两者若带伴奏需要勾选分离选项",66    ),67	tip_p2=dict(68        en="Pitch shift: When there is a large pitch range difference between the Prompt and Target audio, you can try enabling auto pitch shift or manually adjusting the pitch shift in semitones. When a non-zero pitch shift is specified, auto pitch shift will not take effect. The accompaniment of auto mix will be pitch-shifted together with the vocal (keeping the same octave).",69        zh="变调：Prompt 音频的音域和 Target 音频的音域差距较大的时候，可以尝试开启自动变调或手动调整变调半音数，指定非0的变调半音数时，自动变调不生效，自动混音的伴奏会配合歌声进行升降调（保持同一个八度）",70    ),71	tip_p3=dict(72        en="Model parameters: Generally, a larger number of sampling steps will yield better generation quality but also longer generation time; a larger cfg scale will increase timbre similarity and melody fidelity, but may cause more distortion, it is recommended to take a value between 1 and 3.",73        zh="模型参数：一般采样步数越大，生成质量越好，但生成时间也越长；一般cfg系数越大，音色相似度和旋律保真度越高，但是会造成更多的失真，建议取1～3之间的值",74    ),75	tip_p4=dict(76        en="If you want to convert a long audio or a whole song with large pitch range, there may be instability in the generated voice. You can try converting in segments.",77        zh="长音频或完整歌曲中，音域变化较大的情况有可能出现音色不稳定，可以尝试分段转换",78    )79)80 81_GLOBAL_LANG: Literal["zh", "en"] = "zh"82 83 84def _i18n(key: str) -> str:85	return _I18N[key][_GLOBAL_LANG]86 87 88def _print_exception(context: str) -> None:89	print(f"[{context}]\n{traceback.format_exc()}", file=sys.stderr, flush=True)90 91 92def _get_device() -> str:93	return "cuda:0" if torch.cuda.is_available() else "cpu"94 95 96def _session_dir() -> Path:97	timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")98	return ROOT / "outputs" / "gradio" / "svc" / timestamp99 100 101def _normalize_audio_input(audio):102	return audio[0] if isinstance(audio, tuple) else audio103 104 105def _trim_and_save_audio(src_audio_path: str, dst_wav_path: Path, max_sec: int, sr: int = SAMPLE_RATE) -> None:106	audio_data, _ = librosa.load(src_audio_path, sr=sr, mono=True)107	audio_data = audio_data[: max_sec * sr]108	dst_wav_path.parent.mkdir(parents=True, exist_ok=True)109	sf.write(dst_wav_path, audio_data, sr)110 111 112def _usage_md() -> str:113	return "\n\n".join([114		f"### {_i18n('instruction_title')}",115		f"**1.** {_i18n('instruction_p1')}",116		f"**2.** {_i18n('instruction_p2')}",117	])118 119 120def _tips_md() -> str:121	return "\n\n".join([122		f"### {_i18n('tips_title')}",123		f"- {_i18n('tip_p1')}",124		f"- {_i18n('tip_p2')}",125		f"- {_i18n('tip_p3')}",126		f"- {_i18n('tip_p4')}",127	])128 129 130class AppState:131	def __init__(self) -> None:132		self.device = _get_device()133		self.preprocess_pipeline = PreprocessPipeline(134			device=self.device,135			language="Mandarin",136			save_dir=str(ROOT / "outputs" / "gradio" / "_placeholder" / "svc"),137			vocal_sep=True,138			max_merge_duration=60000,139			midi_transcribe=False,140		)141 142		self.svc_config = load_config("soulxsinger/config/soulxsinger.yaml")143		self.svc_model = build_svc_model(144			model_path="pretrained_models/SoulX-Singer/model-svc.pt",145			config=self.svc_config,146			device=self.device,147		)148 149	def run_preprocess(self, audio_path: Path, save_path: Path, vocal_sep: bool) -> tuple[bool, str, Path | None, Path | None]:150		try:151			self.preprocess_pipeline.save_dir = str(save_path)152			self.preprocess_pipeline.run(153				audio_path=str(audio_path),154				vocal_sep=vocal_sep,155				max_merge_duration=60000,156				language="Mandarin",157			)158			vocal_wav = save_path / "vocal.wav"159			vocal_f0 = save_path / "vocal_f0.npy"160			if not vocal_wav.exists() or not vocal_f0.exists():161				return False, f"preprocess output missing: {vocal_wav} or {vocal_f0}", None, None162			return True, "ok", vocal_wav, vocal_f0163		except Exception as e:164			return False, f"preprocess failed: {e}", None, None165 166	def run_svc(167		self,168		prompt_wav_path: Path,169		target_wav_path: Path,170		prompt_f0_path: Path,171		target_f0_path: Path,172		session_base: Path,173		auto_shift: bool,174		auto_mix_acc: bool,175		pitch_shift: int,176		n_step: int,177		cfg: float,178		use_fp16: bool,179		seed: int,180	) -> tuple[bool, str, Path | None]:181		try:182			torch.manual_seed(seed)183			np.random.seed(seed)184			random.seed(seed)185 186			save_dir = session_base / "generated"187			save_dir.mkdir(parents=True, exist_ok=True)188 189			class Args:190				pass191 192			args = Args()193			args.device = self.device194			args.prompt_wav_path = str(prompt_wav_path)195			args.target_wav_path = str(target_wav_path)196			args.prompt_f0_path = str(prompt_f0_path)197			args.target_f0_path = str(target_f0_path)198			args.save_dir = str(save_dir)199			args.auto_shift = auto_shift200			args.pitch_shift = int(pitch_shift)201			args.n_steps = int(n_step)202			args.cfg = float(cfg)203			args.use_fp16 = bool(use_fp16)204 205			svc_process(args, self.svc_config, self.svc_model)206 207			generated = save_dir / "generated.wav"208			if not generated.exists():209				return False, f"inference finished but output not found: {generated}", None210 211			if auto_mix_acc:212				acc_path = session_base / "transcriptions" / "target" / "acc.wav"213				if acc_path.exists():214					vocal_shift = args.pitch_shift215					mul = -1 if vocal_shift < 0 else 1216					acc_shift = abs(vocal_shift) % 12217					acc_shift = mul * acc_shift218					if acc_shift > 6:219						acc_shift -= 12220					if acc_shift < -6:221						acc_shift += 12222 223					mix_sr = self.svc_config.audio.sample_rate224					vocal, _ = librosa.load(str(generated), sr=mix_sr, mono=True)225					acc, _ = librosa.load(str(acc_path), sr=mix_sr, mono=True)226					if acc_shift != 0:227						acc = librosa.effects.pitch_shift(acc, sr=mix_sr, n_steps=acc_shift)228						print(f"Applied pitch shift of {acc_shift} semitones to accompaniment to match vocal shift of {vocal_shift} semitones.")229						230					mix_len = min(len(vocal), len(acc))231					if mix_len > 0:232						mixed = vocal[:mix_len] + acc[:mix_len]233						peak = float(np.max(np.abs(mixed))) if mixed.size > 0 else 1.0234						if peak > 1.0:235							mixed = mixed / peak236						mixed_path = save_dir / "generated_mixed.wav"237						sf.write(str(mixed_path), mixed, mix_sr)238						generated = mixed_path239 240			return True, "svc inference done", generated241		except Exception as e:242			return False, f"svc inference failed: {e}", None243 244 245APP_STATE = AppState()246 247 248@spaces.GPU249def _run_svc_preprocess(250    prompt_audio,251    target_audio,252    prompt_vocal_sep=False,253    target_vocal_sep=True,254):255	try:256		prompt_audio = _normalize_audio_input(prompt_audio)257		target_audio = _normalize_audio_input(target_audio)258		if not prompt_audio or not target_audio:259			gr.Warning(_i18n("warn_missing_audio"))260			return None261 262		session_base = _session_dir()263		audio_dir = session_base / "audio"264		prompt_raw = audio_dir / "prompt.wav"265		target_raw = audio_dir / "target.wav"266		_trim_and_save_audio(prompt_audio, prompt_raw, PROMPT_MAX_SEC_DEFAULT)267		_trim_and_save_audio(target_audio, target_raw, TARGET_MAX_SEC_DEFAULT)268 269		prompt_ok, prompt_msg, prompt_wav, prompt_f0 = APP_STATE.run_preprocess(270			audio_path=prompt_raw,271			save_path=session_base / "transcriptions" / "prompt",272			vocal_sep=bool(prompt_vocal_sep),273		)274		if not prompt_ok or prompt_wav is None or prompt_f0 is None:275			print(prompt_msg, file=sys.stderr, flush=True)276			return None277 278		target_ok, target_msg, target_wav, target_f0 = APP_STATE.run_preprocess(279			audio_path=target_raw,280			save_path=session_base / "transcriptions" / "target",281			vocal_sep=bool(target_vocal_sep),282		)283		if not target_ok or target_wav is None or target_f0 is None:284			print(target_msg, file=sys.stderr, flush=True)285			return None286 287		return (288			str(session_base),289			str(prompt_wav),290			str(prompt_f0),291			str(target_wav),292			str(target_f0),293		)294	except Exception:295		_print_exception("_run_svc_preprocess")296		return None297	finally:298		gc.collect()299		if torch.cuda.is_available():300			torch.cuda.empty_cache()301 302 303@spaces.GPU304def _run_svc_convert(305    preprocess_state,306    auto_shift=True,307    auto_mix_acc=True,308    pitch_shift=0,309    n_step=32,310    cfg=1.0,311    use_fp16=True,312    seed=42,313):314	try:315		if preprocess_state is None or not isinstance(preprocess_state, (tuple, list)) or len(preprocess_state) != 5:316			return None317		session_base_str, prompt_wav, prompt_f0, target_wav, target_f0 = preprocess_state318		session_base = Path(session_base_str)319 320		ok, msg, generated = APP_STATE.run_svc(321			prompt_wav_path=Path(prompt_wav),322			target_wav_path=Path(target_wav),323			prompt_f0_path=Path(prompt_f0),324			target_f0_path=Path(target_f0),325			session_base=session_base,326			auto_shift=bool(auto_shift),327			auto_mix_acc=bool(auto_mix_acc),328			pitch_shift=int(pitch_shift),329			n_step=int(n_step),330			cfg=float(cfg),331			use_fp16=bool(use_fp16),332			seed=int(seed),333		)334		if not ok or generated is None:335			print(msg, file=sys.stderr, flush=True)336			return None337		return str(generated)338	except Exception:339		_print_exception("_run_svc_convert")340		return None341	finally:342		gc.collect()343		if torch.cuda.is_available():344			torch.cuda.empty_cache()345 346 347@spaces.GPU348def _start_svc(349    prompt_audio,350    target_audio,351    prompt_vocal_sep=False,352    target_vocal_sep=True,353    auto_shift=True,354    auto_mix_acc=True,355    pitch_shift=0,356    n_step=32,357    cfg=1.0,358    use_fp16=True,359    seed=42,360):361	state = _run_svc_preprocess(prompt_audio, target_audio, prompt_vocal_sep, target_vocal_sep)362	if state is None:363		return None364	return _run_svc_convert(state, auto_shift, auto_mix_acc, pitch_shift, n_step, cfg, use_fp16, seed)365 366 367def render_tab_content() -> None:368    with gr.Row(equal_height=False):369        # ── Left column: inputs & controls ──370        with gr.Column(scale=1):371            prompt_audio = gr.Audio(372                label="Prompt audio (reference voice)",373                type="filepath",374                interactive=True,375            )376            target_audio = gr.Audio(377                label="Target audio (to convert)",378                type="filepath",379                interactive=True,380            )381 382            run_btn = gr.Button(383                value="🎤 Singing Voice Conversion",384                variant="primary",385                size="lg",386            )387 388            with gr.Accordion("Advanced settings", open=False):389                with gr.Row():390                    prompt_vocal_sep = gr.Checkbox(label="Prompt vocal separation", value=False, scale=1)391                    target_vocal_sep = gr.Checkbox(label="Target vocal separation", value=True, scale=1)392                with gr.Row():393                    auto_shift = gr.Checkbox(label="Auto pitch shift", value=True, scale=1)394                    auto_mix_acc = gr.Checkbox(label="Auto mix accompaniment", value=True, scale=1)395                with gr.Row():396                    use_fp16 = gr.Checkbox(label="Use FP16", value=True, scale=1)397                pitch_shift = gr.Slider(label="Pitch shift (semitones)", value=0, minimum=-36, maximum=36, step=1)398                n_step = gr.Slider(label="diffusion steps", value=32, minimum=1, maximum=200, step=1)399                cfg = gr.Slider(label="cfg scale", value=1.0, minimum=0.0, maximum=10.0, step=0.1)400                seed_input = gr.Slider(label="Seed", value=42, minimum=0, maximum=10000, step=1)401 402        # ── Right column: output ──403        with gr.Column(scale=1):404            output_audio = gr.Audio(label="Generated audio", type="filepath", interactive=False)405            svc_state = gr.State(value=None)406            gr.Examples(407                examples=EXAMPLE_LIST,408                inputs=[prompt_audio, target_audio],409                outputs=[output_audio],410                fn=_start_svc,411                cache_examples=True,412                cache_mode="lazy",413            )414 415    run_btn.click(416        fn=_run_svc_preprocess,417        inputs=[prompt_audio, target_audio, prompt_vocal_sep, target_vocal_sep],418        outputs=[svc_state],419    ).then(420        fn=_run_svc_convert,421        inputs=[svc_state, auto_shift, auto_mix_acc, pitch_shift, n_step, cfg, use_fp16, seed_input],422        outputs=[output_audio],423    )424 425 426def render_interface() -> gr.Blocks:427    with gr.Blocks(title="SoulX-Singer", theme=gr.themes.Default()) as page:428        gr.HTML(429            '<div style="'430            'text-align: center; '431            'padding: 1.25rem 0 1.5rem; '432            'margin-bottom: 0.5rem;'433            '">'434            '<div style="'435            'display: inline-block; '436            'font-size: 1.75rem; '437            'font-weight: 700; '438            'letter-spacing: 0.02em; '439            'line-height: 1.3;'440            '">SoulX-Singer</div>'441            '<div style="'442            'width: 80px; '443            'height: 3px; '444            'margin: 1rem auto 0; '445            'background: linear-gradient(90deg, transparent, #6366f1, transparent); '446            'border-radius: 2px;'447            '"></div>'448            '</div>'449        )450        render_tab_content()451    return page452 453 454if __name__ == "__main__":455	import argparse456 457	parser = argparse.ArgumentParser()458	parser.add_argument("--port", type=int, default=7861, help="Gradio server port")459	parser.add_argument("--share", action="store_true", help="Create public link")460	args = parser.parse_args()461 462	page = render_interface()463	page.queue()464	page.launch(share=args.share, server_name="0.0.0.0", server_port=args.port)465