NoorAsghar/Multi-functional-Chatbot

1import os2import gradio as gr3import torch4from transformers import pipeline, AutoProcessor, BlipForConditionalGeneration5from PIL import Image6from diffusers import StableDiffusionPipeline7import whisper8from gtts import gTTS9import io10import soundfile as sf11from datasets import load_dataset12import requests13from groq import Groq14from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech15 16import google.generativeai as genai17 18# Initialize models and APIs19text_to_image_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")20text_to_image_pipe = text_to_image_pipe.to("cpu")  # Use CPU21 22img_processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-large")23img_caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")24 25whisper_model = whisper.load_model("base")26client = Groq(api_key="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")27 28Google_api_key = os.getenv("AIzaSyDjqiapSQYWaBtsRVVQkQt9tsYgrYuc_wo")29genai.configure(api_key=Google_api_key)30model = genai.GenerativeModel("gemini-1.5-flash")31 32synthesizer = pipeline("text-to-speech", "microsoft/speecht5_tts")33embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")34 35# Define functions36def text_to_text(user_input):37    response = model.generate_content(user_input)38    return response.text39 40# Text-to-Image Generation41def text_to_image(prompt):42    try:43        result = text_to_image_pipe(prompt)44        if hasattr(result, 'images') and len(result.images) > 0:45            image = result.images[0]46            return image47        else:48            return "No image was generated."49    except Exception as e:50        return f"An error occurred: {e}"51 52def image_to_text(image_url):53    image = Image.open(requests.get(image_url, stream=True).raw)54    inputs = img_processor(images=image, text="A picture of", return_tensors="pt")55    outputs = img_caption_model.generate(**inputs)56    caption = img_processor.decode(outputs[0], skip_special_tokens=True)57    return caption58 59def text_to_speech(text, speaker_id):60    try:61        speaker_embedding = torch.tensor(embeddings_dataset[int(speaker_id)]["xvector"]).unsqueeze(0)62        speech = synthesizer(text, forward_params={"speaker_embeddings": speaker_embedding})63        output_path = "output_speech.wav"64        sf.write(output_path, speech["audio"], samplerate=speech["sampling_rate"])65        return output_path66    except Exception as e:67        return f"An error occurred: {e}"68 69def speech_to_speech(file_path):70    try:71        # Transcribe the audio using Whisper72        audio = whisper.load_audio(file_path)73        result = whisper_model.transcribe(audio)74        text = result["text"]75        76        # Generate response using Groq's model77        chat_completion = client.chat.completions.create(78            messages=[{"role": "user", "content": text}],79            model="llama3-8b-8192",80        )81        response_message = chat_completion.choices[0].message.content.strip()82        83        # Convert the response to speech using gTTS84        tts = gTTS(response_message)85        response_audio_io = io.BytesIO()86        tts.write_to_fp(response_audio_io)87        response_audio_io.seek(0)88        89        # Save the audio response to a file90        with open("response.mp3", "wb") as audio_file:91            audio_file.write(response_audio_io.getvalue())92        93        return response_message, "response.mp3"94    except Exception as e:95        return f"An error occurred: {e}", None96 97# Gradio Interface98with gr.Blocks() as demo:99    with gr.Tab("Text to Image"):100        prompt_input = gr.Textbox(label="Describe the image")101        image_output = gr.Image(label="Generated Image")102        image_btn = gr.Button("Generate Image")103        image_btn.click(text_to_image, inputs=prompt_input, outputs=image_output)104 105    with gr.Tab("Image to Text"):106        img_url_input = gr.Textbox(label="Image URL")107        caption_output = gr.Textbox(label="Generated Caption")108        caption_btn = gr.Button("Generate Caption")109        caption_btn.click(image_to_text, inputs=img_url_input, outputs=caption_output)110 111    with gr.Tab("Text to Speech"):112        tts_input = gr.Textbox(label="Enter text")113        speaker_id_input = gr.Number(label="Speaker ID (e.g., 7306)", value=7306)114        tts_output = gr.Audio(label="Generated Speech")115        tts_btn = gr.Button("Convert to Speech")116        tts_btn.click(text_to_speech, inputs=[tts_input, speaker_id_input], outputs=tts_output)117 118    with gr.Tab("Speech to Speech"):119        audio_input = gr.Audio(label="Upload your speech", type="filepath")120        response_output = gr.Textbox(label="Generated Response")121        response_audio_output = gr.Audio(label="Response Audio")122        speech_btn = gr.Button("Generate Response")123        speech_btn.click(speech_to_speech, inputs=audio_input, outputs=[response_output, response_audio_output])124 125demo.launch()126