Group17WPIMLDO24/Case-Study-1
0
1# external imports2import gc3import logging as log4import time5import uuid6import gradio as gr7 8# local imports9from blip_image_caption_large import Blip_Image_Caption_Large10from phi3_mini_4k_instruct import Phi3_Mini_4k_Instruct11from musicgen_small import Musicgen_Small12import config13 14log.basicConfig(level=log.INFO)15 16 17class Image_To_Music:18 def __init__(self, use_local_caption=False, use_local_llm=False, use_local_musicgen=False):19 20 self.use_local_llm = use_local_llm21 self.use_local_caption = use_local_caption22 self.use_local_musicgen = use_local_musicgen23 24 self.image_path = None25 self.generated_caption = None26 self.generated_description = None27 self.audio_path = config.AUDIO_DIR + str(uuid.uuid4()) + ".wav"28 29 self.caption_generation_duration = -130 self.description_generation_duration = -131 self.music_generation_duration = -132 33 def caption_image(self, image_path):34 log.info("Captioning Image...")35 caption_start_time = time.time()36 37 # load model38 self.image_caption_model = Blip_Image_Caption_Large()39 40 self.image_path = image_path41 self.generated_caption = self.image_caption_model.caption_image(self.image_path, self.use_local_caption)42 43 # delete model to free up ram44 del self.image_caption_model45 gc.collect()46 47 self.caption_generation_duration = time.time() - caption_start_time48 log.info(f"Captioning Complete in {self.caption_generation_duration:.2f} seconds: {self.generated_caption} - used local model: {self.use_local_caption}")49 return self.generated_caption50 51 def generate_description(self):52 log.info("Generating Music Description...")53 description_start_time = time.time()54 55 # load model56 self.text_generation_model = Phi3_Mini_4k_Instruct()57 58 messages = [59 {"role": "system", "content": "You are an image caption to song description converter with a deep understanding of Music and Art. You are given the caption of an image. Your task is to generate a textual description of a musical piece that fits the caption. The description should be detailed and vivid, and should include the genre, mood, instruments, tempo, and other relevant information about the music. You should also use your knowledge of art and visual aesthetics to create a musical piece that complements the image. Only output the description of the music, without any explanation or introduction. Be concise."},60 {"role": "user", "content": self.generated_caption},61 ]62 self.generated_description = self.text_generation_model.generate_text(messages, self.use_local_llm)63 64 # delete model to free up ram65 del self.text_generation_model66 gc.collect()67 68 self.description_generation_duration = time.time() - description_start_time69 log.info(f"Description Generation Complete in {self.description_generation_duration:.2f} seconds: {self.generated_description} - used local model: {self.use_local_llm}")70 return self.generated_description71 72 def generate_music(self):73 log.info("Generating Music...")74 music_start_time = time.time()75 76 # load model77 self.music_generation_model = Musicgen_Small()78 79 self.music_generation_model.generate_music(self.generated_description, self.audio_path, self.use_local_musicgen)80 81 # delete model to free up ram82 del self.music_generation_model83 gc.collect()84 85 self.music_generation_duration = time.time() - music_start_time86 log.info(f"Music Generation Complete in {self.music_generation_duration:.2f} seconds: {self.audio_path} - used local model: {self.use_local_musicgen}")87 return self.audio_path88 89 def get_durations(self):90 return f"Caption Generation Time: {self.caption_generation_duration:.2f} seconds\nDescription Generation Time: {self.description_generation_duration:.2f} seconds\nMusic Generation Time: {self.music_generation_duration:.2f} seconds\nTotal Time: {self.caption_generation_duration + self.description_generation_duration + self.music_generation_duration:.2f} seconds"91 92 def run_yield(self, image_path):93 94 self.caption_image(image_path)95 yield [self.generated_caption, None, None, None]96 self.generate_description()97 yield [self.generated_caption, self.generated_description, None, None]98 self.generate_music()99 yield [self.generated_caption, self.generated_description, self.audio_path, None]100 return [self.generated_caption, self.generated_description, self.audio_path,self.get_durations()]101 102 def run(self, image_path):103 self.caption_image(image_path)104 self.generate_description()105 self.generate_music()106 return [self.generated_caption, self.generated_description, self.audio_path, self.get_durations()]107 108 109def run_image_to_music(image_path, llm_max_new_tokens, llm_temperature, llm_top_p, musicgen_max_seconds, use_local_caption, use_local_llm, use_local_musicgen):110 config.LLM_MAX_NEW_TOKENS = llm_max_new_tokens111 config.LLM_TEMPERATURE = llm_temperature112 config.LLM_TOP_P = llm_top_p113 config.MUSICGEN_MAX_NEW_TOKENS = musicgen_max_seconds * 51114 itm = Image_To_Music(use_local_caption=use_local_caption, use_local_llm=use_local_llm, use_local_musicgen=use_local_musicgen)115 return itm.run(image_path)116 117# Gradio UI 118def gradio():119 # Define Gradio Interface, information from (https://www.gradio.app/docs/chatinterface)120 with gr.Blocks() as demo:121 gr.Markdown("<h1 style='text-align: center;'> ⛺ Image to Music Generator 🎼</h1>")122 image_input = gr.Image(type="filepath", label="Upload Image")123 124 125 # ----ATTRIBUTION-START----126 # LLM: ChatGPT4o127 # PROMPT: i need 3 checkbox fields that pass booleans to the run_image_to_music function. it should be "Use local Image Captioning" "Use local LLM" "Use local Music Generation". please make it a nice parameter selector128 # EDITS: /129 130 # Checkbox parameters131 with gr.Row():132 local_captioning = gr.Checkbox(label="Use local Image Captioning", value=False)133 local_llm = gr.Checkbox(label="Use local LLM", value=False)134 local_music_gen = gr.Checkbox(label="Use local Music Generation", value=False)135 # -----ATTRIBUTION-END-----136 137 # ----ATTRIBUTION-START----138 # LLM: ChatGPT4o139 # PROMPT: Now, I need sliders for the different models that are used in the product:\n LLM_MAX_NEW_TOKENS = 50\nLLM_TEMPERATURE = 0.7\nLLM_TOP_P = 0.95\nMUSICGEN_MAX_NEW_TOKENS = 256 # 256 = 5 seconds of audio\n they should be in a hidden menu that opens when I click on "advanced options"\nPlease label them for the end user and fit them nicely in the following UI: <code>140 # EDITS: added interactive flags141 # Advanced options with sliders142 with gr.Accordion("Advanced Options", open=False):143 gr.Markdown("<h3>LLM Settings</h3>")144 llm_max_new_tokens = gr.Slider(1, 200, value=50, step=1, label="LLM Max Tokens", interactive=True)145 llm_temperature = gr.Slider(0.0, 1.0, value=0.7, step=0.01, label="LLM Temperature", interactive=True)146 llm_top_p = gr.Slider(0.01, 0.99, value=0.95, step=0.01, label="LLM Top P", interactive=True)147 148 gr.Markdown("<h3>Music Generation Settings</h3>")149 musicgen_max_seconds = gr.Slider(1, 30, value=5, step=1, label="MusicGen Duration in Seconds (local model only)", interactive=True)150 # -----ATTRIBUTION-END-----151 152 with gr.Row():153 caption_output = gr.Textbox(label="Image Caption")154 music_description_output = gr.Textbox(label="Music Description")155 durations = gr.Textbox(label="Processing Times", interactive=False, placeholder="Time statistics will appear here")156 157 music_output = gr.Audio(label="Generated Music")158 # Button to trigger the process159 generate_button = gr.Button("Generate Music")160 generate_button.click(fn=run_image_to_music, inputs=[image_input, llm_max_new_tokens, llm_temperature, llm_top_p, musicgen_max_seconds, local_captioning, local_llm, local_music_gen], outputs=[caption_output, music_description_output, music_output, durations])161 # Launch Gradio app162 demo.launch(server_port=config.SERVICE_PORT, server_name=config.SERVER_NAME)163 164gradio()165