dskill/DiffRhythm
2
1import gradio as gr2from openai import OpenAI3import requests4import json5# from volcenginesdkarkruntime import Ark6import torch7import torchaudio8from einops import rearrange9import argparse10import json11import os12import spaces13from tqdm import tqdm14import random15import numpy as np16import sys17import base6418from diffrhythm.infer.infer_utils import (19 get_reference_latent,20 get_lrc_token,21 get_audio_style_prompt,22 get_text_style_prompt,23 prepare_model,24 get_negative_style_prompt25)26from diffrhythm.infer.infer import inference27 28MAX_SEED = np.iinfo(np.int32).max29device='cuda'30cfm, tokenizer, muq, vae = prepare_model(device)31cfm = torch.compile(cfm)32 33@spaces.GPU(duration=20)34def infer_music(lrc, text_prompt, seed=42, randomize_seed=False, steps=32, cfg_strength=4.0, file_type='wav', odeint_method='euler', max_frames=2048, device='cuda'):35 36 if randomize_seed:37 seed = random.randint(0, MAX_SEED)38 torch.manual_seed(seed)39 sway_sampling_coef = -1 if steps < 32 else None40 vocal_flag = False41 try:42 lrc_prompt, start_time = get_lrc_token(lrc, tokenizer, device)43 style_prompt = get_text_style_prompt(muq, text_prompt)44 except Exception as e:45 raise gr.Error(f"Error: {str(e)}")46 negative_style_prompt = get_negative_style_prompt(device)47 latent_prompt = get_reference_latent(device, max_frames)48 generated_song = inference(cfm_model=cfm, 49 vae_model=vae, 50 cond=latent_prompt, 51 text=lrc_prompt, 52 duration=max_frames, 53 style_prompt=style_prompt,54 negative_style_prompt=negative_style_prompt,55 steps=steps,56 cfg_strength=cfg_strength,57 sway_sampling_coef=sway_sampling_coef,58 start_time=start_time,59 file_type=file_type,60 vocal_flag=vocal_flag,61 odeint_method=odeint_method,62 )63 return generated_song64 65def R1_infer1(theme, tags_gen, language):66 try:67 client = OpenAI(api_key=os.getenv('HS_DP_API'), base_url = "https://ark.cn-beijing.volces.com/api/v3")68 69 llm_prompt = """70 请围绕"{theme}"主题生成一首符合"{tags}"风格的语言为{language}的完整歌词。严格遵循以下要求：71 72 ### **强制格式规则**73 1. **仅输出时间戳和歌词**，禁止任何括号、旁白、段落标记（如副歌、间奏、尾奏等注释）。74 2. 每行格式必须为 `[mm:ss.xx]歌词内容`，时间戳与歌词间无空格，歌词内容需完整连贯。75 3. 时间戳需自然分布，**第一句歌词起始时间不得为 [00:00.00]**，需考虑前奏空白。76 77 ### **内容与结构要求**78 1. 歌词应富有变化，使情绪递进，整体连贯有层次感。**每行歌词长度应自然变化**，切勿长度一致，导致很格式化。79 2. **时间戳分配应根据歌曲的标签、歌词的情感、节奏来合理推测**，而非机械地按照歌词长度分配。80 3. 间奏/尾奏仅通过时间空白体现（如从 [02:30.00] 直接跳至 [02:50.00]），**无需文字描述**。81 82 ### **负面示例（禁止出现）**83 - 错误：[01:30.00](钢琴间奏)84 - 错误：[02:00.00][副歌]85 - 错误：空行、换行符、注释86 """87 88 response = client.chat.completions.create(89 model="ep-20250304144033-nr9wl",90 messages=[91 {"role": "system", "content": "You are a professional musician who has been invited to make music-related comments."},92 {"role": "user", "content": llm_prompt.format(theme=theme, tags=tags_gen, language=language)},93 ],94 stream=False95 )96 97 info = response.choices[0].message.content98 99 return info100 101 except requests.exceptions.RequestException as e:102 print(f'请求出错: {e}')103 return {}104 105 106 107def R1_infer2(tags_lyrics, lyrics_input):108 client = OpenAI(api_key=os.getenv('HS_DP_API'), base_url = "https://ark.cn-beijing.volces.com/api/v3")109 110 llm_prompt = """111 {lyrics_input}这是一首歌的歌词,每一行是一句歌词,{tags_lyrics}是我希望这首歌的风格，我现在想要给这首歌的每一句歌词打时间戳得到LRC，我希望时间戳分配应根据歌曲的标签、歌词的情感、节奏来合理推测，而非机械地按照歌词长度分配。第一句歌词的时间戳应考虑前奏长度，避免歌词从 `[00:00.00]` 直接开始。严格按照 LRC 格式输出歌词，每行格式为 `[mm:ss.xx]歌词内容`。最后的结果只输出LRC,不需要其他的解释。112 """113 114 response = client.chat.completions.create(115 model="ep-20250304144033-nr9wl",116 messages=[117 {"role": "system", "content": "You are a professional musician who has been invited to make music-related comments."},118 {"role": "user", "content": llm_prompt.format(lyrics_input=lyrics_input, tags_lyrics=tags_lyrics)},119 ],120 stream=False121 )122 123 info = response.choices[0].message.content124 125 return info126 127css = """128/* 固定文本域高度并强制滚动条 */129.lyrics-scroll-box textarea {130 height: 405px !important; /* 固定高度 */131 max-height: 500px !important; /* 最大高度 */132 overflow-y: auto !important; /* 垂直滚动 */133 white-space: pre-wrap; /* 保留换行 */134 line-height: 1.5; /* 行高优化 */135}136 137.gr-examples {138 background: transparent !important;139 border: 1px solid #e0e0e0 !important;140 border-radius: 8px;141 margin: 1rem 0 !important;142 padding: 1rem !important;143}144 145"""146 147 148with gr.Blocks(css=css) as demo:149 gr.HTML(f"""150 <div style="display: flex; align-items: center;">151 <img src='https://raw.githubusercontent.com/ASLP-lab/DiffRhythm/refs/heads/main/src/DiffRhythm_logo.jpg' 152 style='width: 200px; height: 40%; display: block; margin: 0 auto 20px;'>153 </div>154 155 <div style="flex: 1; text-align: center;">156 <div style="font-size: 2em; font-weight: bold; text-align: center; margin-bottom: 5px">157 Di♪♪Rhythm (谛韵)158 </div>159 <div style="display:flex; justify-content: center; column-gap:4px;">160 <a href="https://arxiv.org/abs/2503.01183">161 <img src='https://img.shields.io/badge/Arxiv-Paper-blue'>162 </a> 163 <a href="https://github.com/ASLP-lab/DiffRhythm">164 <img src='https://img.shields.io/badge/GitHub-Repo-green'>165 </a> 166 <a href="https://aslp-lab.github.io/DiffRhythm.github.io/">167 <img src='https://img.shields.io/badge/Project-Page-brown'>168 </a>169 </div>170 </div> 171 """)172 173 with gr.Tabs() as tabs:174 175 # page 1176 with gr.Tab("Music Generate", id=0):177 with gr.Row():178 with gr.Column():179 lrc = gr.Textbox(180 label="Lyrics",181 placeholder="Input the full lyrics",182 lines=12,183 max_lines=50,184 elem_classes="lyrics-scroll-box",185 value="""[00:10.00]Moonlight spills through broken blinds\n[00:13.20]Your shadow dances on the dashboard shrine\n[00:16.85]Neon ghosts in gasoline rain\n[00:20.40]I hear your laughter down the midnight train\n[00:24.15]Static whispers through frayed wires\n[00:27.65]Guitar strings hum our cathedral choirs\n[00:31.30]Flicker screens show reruns of June\n[00:34.90]I'm drowning in this mercury lagoon\n[00:38.55]Electric veins pulse through concrete skies\n[00:42.10]Your name echoes in the hollow where my heartbeat lies\n[00:45.75]We're satellites trapped in parallel light\n[00:49.25]Burning through the atmosphere of endless night\n[01:00.00]Dusty vinyl spins reverse\n[01:03.45]Our polaroid timeline bleeds through the verse\n[01:07.10]Telescope aimed at dead stars\n[01:10.65]Still tracing constellations through prison bars\n[01:14.30]Electric veins pulse through concrete skies\n[01:17.85]Your name echoes in the hollow where my heartbeat lies\n[01:21.50]We're satellites trapped in parallel light\n[01:25.05]Burning through the atmosphere of endless night\n[02:10.00]Clockwork gears grind moonbeams to rust\n[02:13.50]Our fingerprint smudged by interstellar dust\n[02:17.15]Velvet thunder rolls through my veins\n[02:20.70]Chasing phantom trains through solar plane\n[02:24.35]Electric veins pulse through concrete skies\n[02:27.90]Your name echoes in the hollow where my heartbeat lies""" 186 )187 188 text_prompt = gr.Textbox(189 label="Text Prompt",190 placeholder="Enter the Text Prompt, eg: emotional piano pop",191 value="Pop Emotional Piano"192 )193 194 with gr.Column():195 with gr.Accordion("Best Practices Guide", open=True):196 gr.Markdown("""1971. **Lyrics Format Requirements**198 - Each line must follow: `[mm:ss.xx]Lyric content`199 - Example of valid format:200 ``` 201 [00:10.00]Moonlight spills through broken blinds202 [00:13.20]Your shadow dances on the dashboard shrine203 ```2042. **Generation Duration Limits**205 - Current version supports maximum **95 seconds** of music generation206 - Total timestamps should not exceed 01:35.00 (95 seconds)2073. **Text Prompt Format**208 - Use descriptive terms for style like "pop", "rock", "jazz"209 - Add emotions like "emotional", "upbeat", "melancholic"210 - Include instruments like "piano", "guitar", "orchestral"2114. **Supported Languages**212 - **Chinese and English**213 - More languages coming soon214 2155. **Others** 216 - If loading audio result is slow, you can select Output Format as mp3 in Advanced Settings. 217 218 """)219 220 lyrics_btn = gr.Button("Generate", variant="primary")221 audio_output = gr.Audio(label="Audio Result", type="filepath", elem_id="audio_output")222 with gr.Accordion("Advanced Settings", open=False):223 seed = gr.Slider(224 label="Seed",225 minimum=0,226 maximum=MAX_SEED,227 step=1,228 value=0,229 )230 randomize_seed = gr.Checkbox(label="Randomize seed", value=True)231 232 steps = gr.Slider(233 minimum=10,234 maximum=100,235 value=32,236 step=1,237 label="Diffusion Steps",238 interactive=True,239 elem_id="step_slider"240 )241 cfg_strength = gr.Slider(242 minimum=1,243 maximum=10,244 value=4.0,245 step=0.5,246 label="CFG Strength",247 interactive=True,248 elem_id="step_slider"249 )250 odeint_method = gr.Radio(["euler", "midpoint", "rk4","implicit_adams"], label="ODE Solver", value="euler") 251 file_type = gr.Dropdown(["wav", "mp3", "ogg"], label="Output Format", value="wav")252 253 254 255 gr.Examples(256 examples=[257 ["Pop Emotional Piano"],258 ["流行 情感 钢琴"],259 ["Indie folk ballad, coming-of-age themes, acoustic guitar picking with harmonica interludes"],260 ["独立民谣, 成长主题, 原声吉他弹奏与口琴间奏"]261 ],262 inputs=[text_prompt], 263 label="Text Examples",264 examples_per_page=4,265 elem_id="text-examples-container" 266 )267 268 gr.Examples(269 examples=[270 ["""[00:10.00]Moonlight spills through broken blinds\n[00:13.20]Your shadow dances on the dashboard shrine\n[00:16.85]Neon ghosts in gasoline rain\n[00:20.40]I hear your laughter down the midnight train\n[00:24.15]Static whispers through frayed wires\n[00:27.65]Guitar strings hum our cathedral choirs\n[00:31.30]Flicker screens show reruns of June\n[00:34.90]I'm drowning in this mercury lagoon\n[00:38.55]Electric veins pulse through concrete skies\n[00:42.10]Your name echoes in the hollow where my heartbeat lies\n[00:45.75]We're satellites trapped in parallel light\n[00:49.25]Burning through the atmosphere of endless night\n[01:00.00]Dusty vinyl spins reverse\n[01:03.45]Our polaroid timeline bleeds through the verse\n[01:07.10]Telescope aimed at dead stars\n[01:10.65]Still tracing constellations through prison bars\n[01:14.30]Electric veins pulse through concrete skies\n[01:17.85]Your name echoes in the hollow where my heartbeat lies\n[01:21.50]We're satellites trapped in parallel light\n[01:25.05]Burning through the atmosphere of endless night\n[02:10.00]Clockwork gears grind moonbeams to rust\n[02:13.50]Our fingerprint smudged by interstellar dust\n[02:17.15]Velvet thunder rolls through my veins\n[02:20.70]Chasing phantom trains through solar plane\n[02:24.35]Electric veins pulse through concrete skies\n[02:27.90]Your name echoes in the hollow where my heartbeat lies"""],271 ["""[00:04.34]Tell me that I'm special\n[00:06.57]Tell me I look pretty\n[00:08.46]Tell me I'm a little angel\n[00:10.58]Sweetheart of your city\n[00:13.64]Say what I'm dying to hear\n[00:17.35]Cause I'm dying to hear you\n[00:20.86]Tell me I'm that new thing\n[00:22.93]Tell me that I'm relevant\n[00:24.96]Tell me that I got a big heart\n[00:27.04]Then back it up with evidence\n[00:29.94]I need it and I don't know why\n[00:34.28]This late at night\n[00:36.32]Isn't it lonely\n[00:39.24]I'd do anything to make you want me\n[00:43.40]I'd give it all up if you told me\n[00:47.42]That I'd be\n[00:49.43]The number one girl in your eyes\n[00:52.85]Your one and only\n[00:55.74]So what's it gon' take for you to want me\n[00:59.78]I'd give it all up if you told me\n[01:03.89]That I'd be\n[01:05.94]The number one girl in your eyes\n[01:11.34]Tell me I'm going real big places\n[01:14.32]Down to earth so friendly\n[01:16.30]And even through all the phases\n[01:18.46]Tell me you accept me\n[01:21.56]Well that's all I'm dying to hear\n[01:25.30]Yeah I'm dying to hear you\n[01:28.91]Tell me that you need me\n[01:30.85]Tell me that I'm loved\n[01:32.90]Tell me that I'm worth it"""],272 ["""[00:04.27]只因你太美 baby\n[00:08.95]只因你实在是太美 baby\n[00:13.99]只因你太美 baby\n[00:18.89]迎面走来的你让我如此蠢蠢欲动\n[00:20.88]这种感觉我从未有\n[00:21.79]Cause I got a crush on you who you\n[00:25.74]你是我的我是你的谁\n[00:28.09]再多一眼看一眼就会爆炸\n[00:30.31]再近一点靠近点快被融化\n[00:32.49]想要把你占为己有 baby\n[00:34.60]不管走到哪里\n[00:35.44]都会想起的人是你 you you\n[00:38.12]我应该拿你怎样\n[00:39.61]Uh 所有人都在看着你\n[00:42.36]我的心总是不安\n[00:44.18]Oh 我现在已病入膏肓\n[00:46.63]Eh oh\n[00:47.84]难道真的因你而疯狂吗\n[00:51.57]我本来不是这种人\n[00:53.59]因你变成奇怪的人\n[00:55.77]第一次呀变成这样的我\n[01:01.23]不管我怎么去否认\n[01:03.21]只因你太美 baby\n[01:11.46]只因你实在是太美 baby\n[01:16.75]只因你太美 baby\n[01:21.09]Oh eh oh\n[01:22.82]现在确认地告诉我\n[01:25.26]Oh eh oh\n[01:27.31]你到底属于谁\n[01:29.98]Oh eh oh\n[01:31.70]现在确认地告诉我\n[01:34.45]Oh eh oh\n[01:36.35]你到底属于谁\n[01:37.65]就是现在告诉我\n[01:40.00]跟着那节奏 缓缓 make wave\n"""]273 ],274 275 inputs=[lrc],276 label="Lrc Examples",277 examples_per_page=3,278 elem_id="lrc-examples-container",279 )280 281 # page 2282 with gr.Tab("Lyrics Generate", id=1):283 with gr.Row():284 with gr.Column():285 with gr.Accordion("Notice", open=False):286 gr.Markdown("**Two Generation Modes:**\n1. Generate from theme & tags\n2. Add timestamps to existing lyrics")287 288 with gr.Group():289 gr.Markdown("### Method 1: Generate from Theme")290 theme = gr.Textbox(label="theme", placeholder="Enter song theme, e.g: Love and Heartbreak")291 tags_gen = gr.Textbox(label="tags", placeholder="Enter song tags, e.g: pop confidence healing")292 language = gr.Radio(["cn", "en"], label="Language", value="en")293 gen_from_theme_btn = gr.Button("Generate LRC (From Theme)", variant="primary")294 295 gr.Examples(296 examples=[297 [298 "Love and Heartbreak", 299 "vocal emotional piano pop",300 "en"301 ],302 [303 "Heroic Epic", 304 "choir orchestral powerful",305 "cn"306 ]307 ],308 inputs=[theme, tags_gen, language],309 label="Examples: Generate from Theme"310 )311 312 with gr.Group(visible=True): 313 gr.Markdown("### Method 2: Add Timestamps to Lyrics")314 tags_lyrics = gr.Textbox(label="tags", placeholder="Enter song tags, e.g: ballad piano slow")315 lyrics_input = gr.Textbox(316 label="Raw Lyrics (without timestamps)",317 placeholder="Enter plain lyrics (without timestamps), e.g:\nYesterday\nAll my troubles...",318 lines=10,319 max_lines=50,320 elem_classes="lyrics-scroll-box"321 )322 323 gen_from_lyrics_btn = gr.Button("Generate LRC (From Lyrics)", variant="primary")324 325 gr.Examples(326 examples=[327 [328 "acoustic folk happy", 329 """I'm sitting here in the boring room\nIt's just another rainy Sunday afternoon"""330 ],331 [332 "electronic dance energetic",333 """We're living in a material world\nAnd I am a material girl"""334 ]335 ],336 inputs=[tags_lyrics, lyrics_input],337 label="Examples: Generate from Lyrics"338 )339 340 341 with gr.Column():342 lrc_output = gr.Textbox(343 label="Generated LRC",344 placeholder="Timed lyrics will appear here",345 lines=57,346 elem_classes="lrc-output",347 show_copy_button=True348 )349 350 # Bind functions351 gen_from_theme_btn.click(352 fn=R1_infer1,353 inputs=[theme, tags_gen, language],354 outputs=lrc_output355 )356 357 gen_from_lyrics_btn.click(358 fn=R1_infer2,359 inputs=[tags_lyrics, lyrics_input],360 outputs=lrc_output361 )362 363 tabs.select(364 lambda s: None, 365 None, 366 None 367 )368 369 lyrics_btn.click(370 fn=infer_music,371 inputs=[lrc, text_prompt, seed, randomize_seed, steps, cfg_strength, file_type, odeint_method],372 outputs=audio_output373 )374 375 376if __name__ == "__main__":377 demo.launch()