PCMS-AI/MSP

1import gradio as gr2from transformers import AutoProcessor, MusicgenForConditionalGeneration3import numpy as np4import torch5from ram import get_transform, inference_tag2text6from ram.models import tag2text7from PIL import Image8 9 10title = "Musicalization System of Painting Demo"11description = "Pui Ching Middle School: Musicalization System of Painting Demo"12 13 14image_size = 38415device = "cuda" if torch.cuda.is_available() else "cpu"16torch.no_grad()17transform = get_transform(image_size=image_size)18tag2text_model = tag2text(pretrained="tag2text_swin_14m.pth", image_size=image_size, vit='swin_b').eval().to(device)19 20 21def generate_music(raw_image, audio_length):22    raw_image = Image.fromarray(raw_image)23    image = transform(raw_image).unsqueeze(0).to(device)24    res = inference_tag2text(image, tag2text_model)25    tags = res[0].strip(' ').replace('  ', ' ')26    caption = res[2]27    print(caption)28 29    processor = AutoProcessor.from_pretrained("facebook/musicgen-small")30    model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")31    32 33 34    inputs = processor(35        text=[caption],36        padding=True,37        return_tensors="pt",38    )39 40    sampling_rate = model.audio_encoder.config.sampling_rate41    frame_rate = model.audio_encoder.config.frame_rate42    max_new_tokens = int(frame_rate * audio_length)43    audio_values = model.generate(**inputs, max_new_tokens=max_new_tokens)44 45    target_dtype = np.int1646    max_range = np.iinfo(target_dtype).max47    audio_values = audio_values[0, 0].numpy()48    return sampling_rate, (audio_values * max_range).astype(np.int16)49 50 51iface = gr.Interface(52    fn=generate_music,53    title=title,54    description=description,55    inputs=[56        gr.Image(label="Painting"),57        gr.Slider(5, 30, value=15, step=1, label="Audio length(sec)")58    ],59    outputs=gr.Audio(label='Generated Music'))60 61iface.launch()