peekaboo/GenerateStory

1import gradio as gr2from transformers import pipeline3import io, base644from PIL import Image5import numpy as np6import tensorflow as tf7import mediapy8import os9import sys10from huggingface_hub import snapshot_download11 12import streamlit as st13import firebase_admin14from firebase_admin import credentials15from firebase_admin import firestore16import datetime17import tempfile18from typing import Optional19import numpy as np20from TTS.utils.manage import ModelManager21from TTS.utils.synthesizer import Synthesizer22             23 24# firestore singleton is a cached multiuser instance to persist shared crowdsource memory25@st.experimental_singleton26def get_db_firestore():27    cred = credentials.Certificate('test.json')28    firebase_admin.initialize_app(cred, {'projectId': u'clinical-nlp-b9117',})29    db = firestore.client()30    return db31 32#start firestore singleton33db = get_db_firestore()34 35# create ASR ML pipeline36asr = pipeline("automatic-speech-recognition", "facebook/wav2vec2-base-960h")37 38# create Text Classification pipeline39classifier = pipeline("text-classification")40 41# create text generator pipeline42story_gen = pipeline("text-generation", "pranavpsv/gpt2-genre-story-generator")43 44# transcribe function45def transcribe(audio):46    text = asr(audio)["text"]47    return text48 49def speech_to_text(speech):50    text = asr(speech)["text"]51    return text52 53def text_to_sentiment(text):54    sentiment = classifier(text)[0]["label"]55    return sentiment 56 57def upsert(text):58    date_time =str(datetime.datetime.today())59    doc_ref = db.collection('Text2SpeechSentimentSave').document(date_time)60    doc_ref.set({u'firefield': 'Recognize Speech', u'first': 'https://huggingface.co/spaces/awacke1/Text2SpeechSentimentSave', u'last': text, u'born': date_time,})61    saved = select('Text2SpeechSentimentSave', date_time)62    # check it here:  https://console.firebase.google.com/u/0/project/clinical-nlp-b9117/firestore/data/~2FStreamlitSpaces63    return saved64      65def select(collection, document):66    doc_ref = db.collection(collection).document(document)67    doc = doc_ref.get()68    docid = ("The id is: ", doc.id)69    contents = ("The contents are: ", doc.to_dict())70    return contents71          72def selectall(text):73    docs = db.collection('Text2SpeechSentimentSave').stream()74    doclist=''75    for doc in docs:76        r=(f'{doc.id} => {doc.to_dict()}')77        doclist += r78    return doclist 79 80# story gen81def generate_story(choice, input_text):82    query = "<BOS> <{0}> {1}".format(choice, input_text)83    generated_text = story_gen(query)84    generated_text = generated_text[0]['generated_text']85    generated_text = generated_text.split('> ')[2]86    return generated_text87    88# images gen89def generate_images(text):90    steps=5091    width=25692    height=25693    num_images=494    diversity=695    image_bytes = image_gen(text, steps, width, height, num_images, diversity)96    generated_images = []97    for image in image_bytes[1]:98        image_str = image[0]99        image_str = image_str.replace("data:image/png;base64,","")100        decoded_bytes = base64.decodebytes(bytes(image_str, "utf-8"))101        img = Image.open(io.BytesIO(decoded_bytes))102        generated_images.append(img)103    return generated_images104    105# reductionism - interpolate 4 images - todo - unhardcode the pattern106def generate_interpolation(gallery):107    times_to_interpolate = 4108    generated_images = []109    for image_str in gallery:110        image_str = image_str.replace("data:image/png;base64,","")111        decoded_bytes = base64.decodebytes(bytes(image_str, "utf-8"))112        img = Image.open(io.BytesIO(decoded_bytes))113        generated_images.append(img)114    generated_images[0].save('frame_0.png')115    generated_images[1].save('frame_1.png')116    generated_images[2].save('frame_2.png')117    generated_images[3].save('frame_3.png')118    input_frames = ["frame_0.png", "frame_1.png", "frame_2.png", "frame_3.png"]119    frames = list(util.interpolate_recursively_from_files(input_frames, times_to_interpolate, interpolator))120    mediapy.write_video("out.mp4", frames, fps=15)121    return "out.mp4"122 123# image generator124image_gen = gr.Interface.load("spaces/multimodalart/latentdiffusion")125 126# video generator127os.system("git clone https://github.com/google-research/frame-interpolation")128sys.path.append("frame-interpolation")129from eval import interpolator, util130 131ffmpeg_path = util.get_ffmpeg_path()132mediapy.set_ffmpeg(ffmpeg_path)133model = snapshot_download(repo_id="akhaliq/frame-interpolation-film-style")134interpolator = interpolator.Interpolator(model, None)135 136demo = gr.Blocks()137with demo:138 139    audio_file = gr.inputs.Audio(source="microphone", type="filepath")140    text = gr.Textbox()141    label = gr.Label()142    saved = gr.Textbox()143    savedAll = gr.Textbox()144    audio = gr.Audio(label="Output", interactive=False)145    146    b1 = gr.Button("Recognize Speech")147    b2 = gr.Button("Classify Sentiment")148    b3 = gr.Button("Save Speech to Text")149    b4 = gr.Button("Retrieve All")150 151    input_story_type = gr.Radio(choices=['superhero', 'action', 'drama', 'horror', 'thriller', 'sci_fi'], value='sci_fi', label="Genre")152    input_start_text = gr.Textbox(placeholder='A teddy bear outer space', label="Starting Text")153    154    gr.Markdown("1. Select a type of story, then write some starting text! Then hit the 'Generate Story' button to generate a story! Feel free to edit the generated story afterwards!")155    button_gen_story = gr.Button("Generate Story")156    gr.Markdown("2. After generating a story, hit the 'Generate Images' button to create some visuals for your story! (Can re-run multiple times!)")157    button_gen_images = gr.Button("Generate Images")158    gr.Markdown("3. After generating some images, hit the 'Generate Video' button to create a short video by interpolating the previously generated visuals!")159    button_gen_video = gr.Button("Generate Video")160    output_generated_story = gr.Textbox(label="Generated Story")161    output_gallery = gr.Gallery(label="Generated Story Images")162    output_interpolation = gr.Video(label="Generated Video")163            164    # Bind functions to buttons165    button_gen_story.click(fn=generate_story, inputs=[input_story_type , input_start_text], outputs=output_generated_story)166    button_gen_images.click(fn=generate_images, inputs=output_generated_story, outputs=output_gallery)167    button_gen_video.click(fn=generate_interpolation, inputs=output_gallery, outputs=output_interpolation)168    169    b1.click(speech_to_text, inputs=audio_file, outputs=input_start_text )170    b2.click(text_to_sentiment, inputs=text, outputs=label)171    b3.click(upsert, inputs=text, outputs=saved)172    b4.click(selectall, inputs=text, outputs=savedAll)173    174demo.launch(debug=True, enable_queue=True)