jcheng5/multimodal

1import base642import tempfile3 4from openai import AsyncOpenAI5from shiny.express import input, render, ui6 7from videoinput import audio_spinner, input_video_clip, process_video8 9client = AsyncOpenAI()10 11ui.page_opts(class_="py-5")12 13input_video_clip("clip")14 15 16@render.ui17async def show_clip():18    clip = input.clip()19    mime_type = clip["type"]20    bytes = base64.b64decode(clip["bytes"])21    # TODO: Use correct file extension based on mime type22    with tempfile.TemporaryDirectory() as tempdir:23        filename = tempfile.mktemp(dir=tempdir, suffix=get_video_extension(mime_type))24        with open(filename, "wb") as file:25            file.write(bytes)26            file.close()27 28        with ui.Progress() as p:29 30            mp3_data_uri = await process_video(31                client,32                filename,33                callback=lambda status: p.set(message=status),34            )35            return audio_spinner(src=mp3_data_uri)36 37 38def get_video_extension(mime_type: str) -> str:39    mime_type = mime_type.split(";")[0].strip()40 41    # Dictionary to map MIME types to file extensions42    mime_to_extension = {43        "video/webm": ".webm",44        "video/mp4": ".mp4",45        "video/ogg": ".ogv",46        "video/x-matroska": ".mkv",47        "video/avi": ".avi",48        "video/mpeg": ".mpeg",49        "video/quicktime": ".mov",50    }51 52    # Return the appropriate file extension for the given MIME type53    return mime_to_extension.get(mime_type, "")54