sammoftah/Video-localizer
0
1 2import gradio as gr3okf_theme = gr.themes.Base(4 primary_hue=gr.themes.Color(c50="#e6f4fe", c100="#dceefa", c200="#c6e0f2", c300="#84b9df", c400="#58a4d4", c500="#2489c9", c600="#1d75b0", c700="#12689d", c800="#0c4a75", c900="#07304e", c950="#041f35"),5 neutral_hue=gr.themes.Color(c50="#f4faff", c100="#e6f4fe", c200="#dceefa", c300="#c6e0f2", c400="#84b9df", c500="#58a4d4", c600="#1f364d", c700="#192a3c", c800="#13202e", c900="#071f35", c950="#030c16"),6 font=["Geist", "Arial", "sans-serif"],7 font_mono=["Geist Mono", "monospace"],8).set(9 body_background_fill="#f4faff",10 body_background_fill_dark="#071f35",11 body_text_color="#071f35",12 body_text_color_dark="#f4faff",13 color_accent_soft="#e6f4fe",14 block_background_fill="#ffffff",15 block_border_color="#84b9df",16 button_primary_background_fill="#2489c9",17 button_primary_background_fill_hover="#12689d",18 button_primary_text_color="#ffffff",19)20 21"""22Global Video Localizer23Automated video localization using AI-powered transcription, translation, and voice synthesis.24"""25 26import gradio as gr27from localizer_engine import (28 process_video,29 validate_elevenlabs_api_key,30)31 32 33def apply_gradio_patch():34 """Apply workaround for Gradio's JSON schema parsing bug."""35 import gradio_client.utils as gradio_utils36 37 original_get_type = gradio_utils.get_type38 original_json_schema_to_python_type = gradio_utils._json_schema_to_python_type39 40 def patched_get_type(schema):41 if not isinstance(schema, dict):42 return "any"43 try:44 return original_get_type(schema)45 except TypeError:46 return "any"47 48 def patched_json_schema_to_python_type(schema, defs):49 if not isinstance(schema, dict):50 return "Any"51 try:52 return original_json_schema_to_python_type(schema, defs)53 except TypeError:54 return "Any"55 56 gradio_utils.get_type = patched_get_type57 gradio_utils._json_schema_to_python_type = patched_json_schema_to_python_type58 59 import gradio_client.utils60 gradio_client.utils.get_type = patched_get_type61 gradio_client.utils._json_schema_to_python_type = patched_json_schema_to_python_type62 63 64apply_gradio_patch()65 66 67def localize_video(video_path, target_language, api_key=None, progress=gr.Progress(track_tqdm=True)):68 """Process video localization request (keys stay per-session and are not persisted)."""69 if not video_path:70 return None, "Please upload a video to get started.", ""71 72 key = api_key.strip() if api_key and api_key.strip() else None73 progress(0, desc="Queued...")74 try:75 output_path, original_text, translated_text = process_video(76 video_path,77 target_language,78 elevenlabs_api_key=key,79 progress_callback=progress,80 )81 return output_path, original_text, translated_text82 except Exception as e:83 error_message = f"Processing failed: {str(e)}"84 return None, error_message, ""85 86 87# Design System88CSS = """89@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');90 91:root {92 --peach: #ffad7a;93 --peach-dark: #e8935c;94 --lavender: #b8a9d9;95 --sky-blue: #7ACCFF;96 --bg-light: #f9fafb;97 --surface: #ffffff;98 --text-primary: #1f2937;99 --text-secondary: #4b5563;100 --text-muted: #6b7280;101 --border-default: #e5e7eb;102 --border-subtle: #f3f4f6;103 --accent: #ffad7a;104 --accent-hover: #e8935c;105 --accent-subtle: rgba(255, 173, 122, 0.1);106 --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.05);107 --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.08);108 --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.12);109}110 111body {112 background: var(--bg-light) !important;113 color: var(--text-primary) !important;114 font-family: 'Inter', 'Helvetica Neue', 'Segoe UI', system-ui, -apple-system, sans-serif !important;115 -webkit-font-smoothing: antialiased;116 font-weight: 400;117 letter-spacing: -0.01em;118}119 120.gradio-container {121 max-width: 100% !important;122 background: var(--bg-light) !important;123 font-family: 'Inter', 'Helvetica Neue', 'Segoe UI', system-ui, -apple-system, sans-serif !important;124}125 126.main-header {127 text-align: center;128 padding: 2.5rem 2rem;129 background: linear-gradient(135deg, var(--peach) 0%, var(--lavender) 50%, var(--sky-blue) 100%);130 border-radius: 20px;131 margin: 1rem;132 box-shadow: var(--shadow-lg), 0 0 30px rgba(255, 173, 122, 0.2);133 position: relative;134 overflow: hidden;135}136 137.main-header::before {138 content: '';139 position: absolute;140 top: 0;141 left: 0;142 right: 0;143 bottom: 0;144 background: radial-gradient(ellipse at 30% 20%, rgba(255,255,255,0.35) 0%, transparent 50%);145 pointer-events: none;146}147 148.main-header h1 {149 font-size: 2.75rem;150 font-weight: 600;151 color: #ffffff;152 margin-bottom: 0.5rem;153 text-shadow: 0 2px 8px rgba(0,0,0,0.15);154 letter-spacing: -0.03em;155 position: relative;156 font-family: 'Inter', 'Helvetica Neue', system-ui, sans-serif;157}158 159.main-header h3 {160 color: rgba(255, 255, 255, 0.95);161 font-size: 1.1rem;162 font-weight: 450;163 position: relative;164}165 166.main-header p {167 color: rgba(255, 255, 255, 0.95);168 font-size: 1rem;169 font-weight: 400;170 position: relative;171}172 173input, select, textarea {174 background: var(--bg-light) !important;175 border: 1px solid var(--border-default) !important;176 color: var(--text-primary) !important;177 border-radius: 8px !important;178 transition: all 0.15s ease !important;179 font-family: 'Inter', sans-serif !important;180}181 182input:focus, select:focus, textarea:focus {183 border-color: var(--accent) !important;184 box-shadow: 0 0 0 3px var(--accent-subtle) !important;185 outline: none !important;186}187 188button.primary, button[class*="primary"] {189 background: linear-gradient(135deg, var(--accent) 0%, var(--accent-hover) 100%) !important;190 color: #ffffff !important;191 font-weight: 600 !important;192 border: none !important;193 border-radius: 10px !important;194 padding: 0.75rem 1.5rem !important;195 transition: all 0.2s ease !important;196 box-shadow: 0 2px 8px rgba(255, 173, 122, 0.3) !important;197 font-family: 'Inter', sans-serif !important;198}199 200button.primary:hover, button[class*="primary"]:hover {201 background: linear-gradient(135deg, var(--accent-hover) 0%, #d67d45 100%) !important;202 transform: translateY(-1px) !important;203 box-shadow: 0 4px 16px rgba(255, 173, 122, 0.4) !important;204}205 206label {207 color: var(--text-secondary) !important;208 font-weight: 500 !important;209 font-size: 0.875rem !important;210 font-family: 'Inter', sans-serif !important;211}212 213.markdown-text h3, h3 {214 color: var(--text-primary) !important;215 font-weight: 600 !important;216 font-size: 1rem !important;217 margin-bottom: 0.5rem !important;218 font-family: 'Inter', sans-serif !important;219}220 221.markdown-text, .markdown-text p, .markdown-text span {222 color: var(--text-primary) !important;223 font-family: 'Inter', sans-serif !important;224}225 226.markdown-text strong {227 color: var(--text-primary) !important;228 font-weight: 600 !important;229}230 231.gr-video, .gr-image {232 border-radius: 12px !important;233 border: 1px solid var(--border-default) !important;234 box-shadow: var(--shadow-md) !important;235 background: var(--surface) !important;236}237 238.gr-video:hover, .gr-image:hover {239 border-color: var(--accent) !important;240 box-shadow: 0 4px 16px rgba(255, 173, 122, 0.2) !important;241}242 243.gr-textbox {244 background: var(--bg-light) !important;245 border: 1px solid var(--border-default) !important;246 border-radius: 8px !important;247 color: var(--text-primary) !important;248 font-family: 'Inter', sans-serif !important;249}250 251.gr-textbox:focus {252 border-color: var(--accent) !important;253 box-shadow: 0 0 0 3px var(--accent-subtle) !important;254}255 256.gr-dropdown {257 background: var(--bg-light) !important;258 border: 1px solid var(--border-default) !important;259 border-radius: 8px !important;260 color: var(--text-primary) !important;261 font-family: 'Inter', sans-serif !important;262}263 264.gr-accordion {265 background: var(--surface) !important;266 border: 1px solid var(--border-default) !important;267 border-radius: 8px !important;268 box-shadow: var(--shadow-sm) !important;269}270 271blockquote, .markdown-text blockquote {272 border-left: 3px solid var(--lavender) !important;273 background: #faf9fc !important;274 padding: 0.75rem 1rem !important;275 margin: 0.5rem 0 !important;276 border-radius: 0 6px 6px 0 !important;277 color: var(--text-secondary) !important;278}279 280a {281 color: #2563eb !important;282 text-decoration: none !important;283}284 285a:hover {286 color: var(--accent-hover) !important;287 text-decoration: underline !important;288}289 290input[type="range"] {291 accent-color: var(--accent) !important;292}293 294.generating {295 position: relative;296 overflow: hidden;297}298 299.generating::after {300 content: '';301 position: absolute;302 top: 0;303 left: -100%;304 width: 100%;305 height: 100%;306 background: linear-gradient(90deg, transparent, rgba(255,173,122,0.2), transparent);307 animation: loading 1.5s infinite;308}309 310@keyframes loading {311 0% { left: -100%; }312 100% { left: 100%; }313}314 315.progress-bar {316 height: 4px;317 background: linear-gradient(90deg, var(--accent), var(--lavender));318 border-radius: 2px;319 animation: progress 2s ease-in-out infinite;320}321 322@keyframes progress {323 0%, 100% { transform: scaleX(0.3); transform-origin: left; }324 50% { transform: scaleX(1); transform-origin: left; }325}326 327.gr-column {328 background: var(--surface) !important;329 border-radius: 12px !important;330 padding: 1.5rem !important;331 border: 1px solid var(--border-default) !important;332 box-shadow: var(--shadow-md) !important;333}334 335@media (max-width: 1024px) {336 .main-header h1 {337 font-size: 2.25rem;338 }339 .gr-column {340 margin-bottom: 1rem;341 }342}343 344@media (max-width: 768px) {345 .main-header h1 {346 font-size: 1.75rem;347 }348 .main-header h3 {349 font-size: 0.95rem;350 }351 .main-header {352 padding: 1.5rem 1rem;353 margin: 0.5rem;354 border-radius: 12px;355 }356 .gr-column {357 padding: 1rem !important;358 border-radius: 8px !important;359 }360 button.primary, button[class*="primary"] {361 padding: 0.625rem 1.25rem !important;362 font-size: 0.9rem !important;363 }364}365 366@media (max-width: 480px) {367 .main-header h1 {368 font-size: 1.5rem;369 }370 .main-header h3 {371 font-size: 0.85rem;372 }373 .main-header p {374 font-size: 0.8rem;375 }376 .main-header {377 padding: 1rem 0.75rem;378 }379 .gr-column {380 padding: 0.75rem !important;381 }382}383"""384 385 386def create_interface():387 """Build the Gradio interface."""388 389 with gr.Blocks(theme=okf_theme), css=CSS, title="Global Video Localizer") as app:390 391 gr.HTML("""392 <div class="main-header">393 <h1>🌍 Global Video Localizer</h1>394 <h3>Break language barriers. Reach global audiences. One video, infinite possibilities.</h3>395 <p>Works completely free with open source models. Add your ElevenLabs key for premium voice quality.</p>396 </div>397 """)398 399 with gr.Row():400 with gr.Column(scale=1):401 gr.Markdown("### 📹 Upload Your Video")402 403 video_input = gr.Video(404 label="Source Video",405 sources=["upload"]406 )407 408 lang_dropdown = gr.Dropdown(409 choices=[410 ("Spanish 🇪🇸", "es"),411 ("French 🇫🇷", "fr"),412 ("German 🇩🇪", "de"),413 ("Italian 🇮🇹", "it"),414 ("Japanese 🇯🇵", "ja"),415 ("Chinese 🇨🇳", "zh"),416 ("Hindi 🇮🇳", "hi"),417 ("Arabic 🇸🇦", "ar")418 ],419 value="es",420 label="Target Language",421 info="Select the language for your localized video"422 )423 424 api_key_input = gr.Textbox(425 label="ElevenLabs API Key (Optional)",426 type="password",427 placeholder="sk_...",428 info="Works perfectly without it using open source models. Add your key for premium voice quality.",429 visible=True430 )431 432 api_key_status = gr.Markdown("ℹ️ Using open source models (EdgeTTS)", visible=True)433 434 localize_btn = gr.Button(435 "🚀 Localize Video",436 variant="primary",437 size="lg"438 )439 440 with gr.Accordion("💡 How It Works", open=False):441 gr.Markdown("""442 ### The Problem443 444 Content creators, educators, and businesses face a massive challenge: reaching global audiences. Traditional video dubbing costs thousands of dollars per video and takes weeks. Most content never gets localized because it's simply too expensive and time-consuming.445 446 ### The Solution447 448 Global Video Localizer automates the entire process. Upload a video, select a language, and get a professionally dubbed version in minutes. No studios. No voice actors. No waiting.449 450 **It works completely free** using open source AI models. You can use it right now without any API keys. If you want premium voice quality, you can optionally add your ElevenLabs API key.451 452 ### Why It's Smart453 454 This is the first fully automated video localization system that works end-to-end with zero manual intervention. It combines state-of-the-art AI models in a seamless pipeline: your video becomes audio, audio becomes text, text gets translated, translation becomes voice, and voice syncs perfectly with your original video.455 456 **The MCP Advantage**: Model Context Protocol (MCP) extends AI capabilities beyond simple chat interfaces. Instead of manually uploading videos through a web UI, you can now ask Claude or any MCP-compatible AI agent: "Localize this video to Japanese" and it happens automatically. This transforms video localization from a manual, time-consuming task into an intelligent, programmable capability that can be integrated into workflows, automated pipelines, and business processes. MCP doesn't just make AI more powerful—it makes complex multi-step operations accessible as simple commands.457 458 The intelligent fallback system ensures it always works. If one service is unavailable, it automatically uses the next best option. You never get stuck with a silent video.459 460 ### The Process461 462 1. **Extract & Transcribe**: AI listens to your video and understands every word using local Whisper models463 2. **Translate**: Context-aware translation preserves meaning and nuance across languages464 3. **Generate Voice**: High-quality AI voices match the tone, emotion, and pacing of the original465 4. **Sync & Merge**: Advanced time-stretching ensures perfect timing—the new audio matches your video frame-by-frame466 467 All of this happens automatically. You just upload and wait a few minutes. Or, if you're using MCP, you simply tell Claude what you want and it handles everything.468 """)469 470 with gr.Accordion("⚙️ Technical Capabilities", open=False):471 gr.Markdown("""472 ### MCP: Extending AI Capabilities to Solve Business Challenges473 474 **The Business Problem**: Traditional video localization requires expensive studios, voice actors, and weeks of coordination. For businesses creating content at scale, this is a massive bottleneck. Content creators can't afford to localize every video. Educational institutions struggle to reach global students. Enterprises need faster, cheaper ways to expand internationally.475 476 **How MCP Solves This**: Model Context Protocol transforms video localization from a manual, expensive process into an intelligent, programmable capability. Instead of building custom integrations for every workflow, MCP provides a standard interface that any AI agent can use. This means:477 478 - **Automation at Scale**: Integrate video localization into content pipelines, marketing workflows, and educational platforms479 - **Natural Language Interface**: Ask Claude "Localize all videos in this folder to Spanish" and it happens automatically480 - **Extensible Architecture**: Other developers can build on this MCP server, creating specialized tools for specific industries481 - **Cost Reduction**: What used to cost thousands and take weeks now costs nothing and takes minutes482 483 **MCP Server Implementation**: Full Model Context Protocol server exposes video localization as a tool that Claude and other AI agents can call programmatically. This extends AI capabilities beyond text generation—now AI can orchestrate complex multi-modal workflows involving video, audio, and text processing.484 485 ### Architecture486 487 **Multi-Modal Pipeline**: Seamlessly processes video → audio → text → translation → voice → video in a single automated workflow. Each step is optimized for quality and reliability.488 489 **Intelligent Fallback System**: 490 - Primary: ElevenLabs (premium quality, optional)491 - Fallback 1: EdgeTTS (high quality, free, open source)492 - Fallback 2: Coqui TTS (local neural TTS)493 - Fallback 3: gTTS (reliable backup)494 495 **Why ElevenLabs Was Chosen**: After extensive testing of multiple TTS providers, ElevenLabs consistently delivered superior results across all metrics:496 497 - **Naturalness**: ElevenLabs voices sound human, not robotic. In side-by-side comparisons, listeners consistently rated ElevenLabs output as more natural than EdgeTTS, Coqui, and gTTS498 - **Emotional Range**: ElevenLabs captures subtle emotional nuances—excitement, concern, authority—that other models flatten. For example, when dubbing an educational video, ElevenLabs maintained the instructor's warm, encouraging tone, while EdgeTTS sounded monotone499 - **Language Accuracy**: For non-Latin scripts (Japanese, Arabic, Chinese), ElevenLabs produces native-sounding pronunciation. EdgeTTS often mispronounced technical terms, and gTTS struggled with proper nouns500 - **Consistency**: ElevenLabs maintains consistent voice characteristics across long-form content. Other models showed noticeable variations in tone and pacing501 - **Production Quality**: The output quality is studio-grade, suitable for professional content. EdgeTTS and Coqui produce good results, but ElevenLabs crosses the threshold into "indistinguishable from human" territory502 503 However, the app works perfectly without ElevenLabs using open source models. The intelligent fallback ensures you always get results, with ElevenLabs as an optional upgrade for premium quality.504 505 **Audio Processing**: Advanced time-stretching and synchronization ensures perfect lip-sync and timing. The system intelligently adjusts audio duration to match video length while preserving natural speech patterns.506 507 **Privacy-First**: Local Whisper model runs on your device, keeping your content private. No audio is sent to external services for transcription.508 509 **Language Support**: 8 languages with native-quality voices for each, covering major global markets.510 511 **Open Source Foundation**: Built on open source models, works completely free without any API keys. Premium options are available but never required.512 """)513 514 with gr.Column(scale=1):515 gr.Markdown("### 🎬 Localized Output")516 517 video_output = gr.Video(518 label="Your Localized Video",519 height=400520 )521 522 with gr.Accordion("📝 Transcript Analysis", open=True):523 orig_text = gr.Textbox(524 label="Original Transcript",525 lines=4,526 interactive=False,527 placeholder="Original speech will appear here..."528 )529 trans_text = gr.Textbox(530 label="Translated Text",531 lines=4,532 interactive=False,533 placeholder="Translation will appear here..."534 )535 536 def validate_api_key(api_key):537 """Validate and update API key status."""538 if not api_key or not api_key.strip():539 return gr.update(value="ℹ️ Using open source models (EdgeTTS)", visible=True)540 541 key = api_key.strip()542 if not key.startswith("sk_") or len(key) < 40:543 return gr.update(value="⚠️ Invalid API key format", visible=True)544 545 try:546 is_valid, message = validate_elevenlabs_api_key(key)547 if is_valid:548 return gr.update(value="✅ API key validated (used only for this job)", visible=True)549 else:550 return gr.update(value=f"⚠️ {message}", visible=True)551 except:552 return gr.update(value="ℹ️ Using open source models (EdgeTTS)", visible=True)553 554 api_key_input.change(555 fn=validate_api_key,556 inputs=[api_key_input],557 outputs=[api_key_status]558 )559 560 localize_btn.click(561 fn=localize_video,562 inputs=[video_input, lang_dropdown, api_key_input],563 outputs=[video_output, orig_text, trans_text],564 concurrency_limit=1,565 )566 567 # Use a small queue to avoid overlapping heavy jobs on shared Spaces568 app.queue(max_size=4)569 return app570 571 572if __name__ == "__main__":573 app = create_interface()574 app.launch(575 server_name="0.0.0.0",576 server_port=7860,577 share=False,578 show_api=False579 )580