awacke1/SentenceToGeneratedVideo

1import gradio as gr2from transformers import pipeline3import io, base644from PIL import Image5import numpy as np6import tensorflow as tf7import mediapy8import os9import sys10from huggingface_hub import snapshot_download11 12image_gen = gr.Interface.load("spaces/multimodalart/latentdiffusion")13 14os.system("git clone https://github.com/google-research/frame-interpolation")15sys.path.append("frame-interpolation")16from eval import interpolator, util17 18ffmpeg_path = util.get_ffmpeg_path()19mediapy.set_ffmpeg(ffmpeg_path)20 21model = snapshot_download(repo_id="akhaliq/frame-interpolation-film-style")22interpolator = interpolator.Interpolator(model, None)23 24def generate_story(choice, input_text):25    query = "<BOS> <{0}> {1}".format(choice, input_text)26    27    print(query)28    generated_text = story_gen(query)29    generated_text = generated_text[0]['generated_text']30    generated_text = generated_text.split('> ')[2]31    32    return generated_text33    34def generate_images(text):35    steps=5036    width=25637    height=25638    num_images=439    diversity=440    image_bytes = image_gen(text, steps, width, height, num_images, diversity)41    42    # Algo from spaces/Gradio-Blocks/latent_gpt2_story/blob/main/app.py43    generated_images = []44    for image in image_bytes[1]:45        image_str = image[0]46        image_str = image_str.replace("data:image/png;base64,","")47        decoded_bytes = base64.decodebytes(bytes(image_str, "utf-8"))48        img = Image.open(io.BytesIO(decoded_bytes))49        generated_images.append(img)50        51    return generated_images52    53def generate_interpolation(text):54    times_to_interpolate = 455    56    generated_images = generate_images(text)57    58    generated_images[0].save('frame_0.png')59    generated_images[1].save('frame_1.png')60    generated_images[2].save('frame_2.png')61    generated_images[3].save('frame_3.png')62    63    input_frames = ["frame_0.png", "frame_1.png", "frame_2.png", "frame_3.png"]64 65    frames = list(util.interpolate_recursively_from_files(input_frames, times_to_interpolate, interpolator))66 67    mediapy.write_video("out.mp4", frames, fps=7)68    69    return "out.mp4"70    71    72 73demo = gr.Blocks()74 75with demo:76    input_start_text = gr.Textbox(placeholder='A yellow face amazon parrot saddles up his horse and goes for a horseback ride across the Amazon river', label="Starting Text")77    button_gen_video = gr.Button("Generate Video")78    output_interpolation = gr.Video(label="Generated Video")79            80    button_gen_video.click(fn=generate_interpolation, inputs=input_start_text, outputs=output_interpolation)81    examples=[["Three yellow nape amazon parrots dance and celebrate a birthday."],["Two horses trot together across a sunset landscape green field"]]82demo.launch(debug=True, enable_queue=True)