sohith0318/Multilingual_corpus_ai_engine
0
1import streamlit as st2import os3import pandas as pd4import zipfile5import io6import random7from PIL import Image8import uuid9from datetime import datetime10import streamlit as st11import speech_recognition as sr12import os13from io import BytesIO14import wave15import uuid16from datetime import datetime17import sounddevice as sd18import soundfile as sf19 20 21class SpeechRecognizer:22 def __init__(self):23 self.recognizer = sr.Recognizer()24 # Ensure voice input directory exists25 if not os.path.exists("voice_inputs"):26 os.makedirs("voice_inputs")27 28 def adjust_for_ambient_noise(self):29 """No-op when using sounddevice for recording (kept for API compatibility)"""30 return31 32 def record_with_sounddevice(self, duration_seconds: int = 10, samplerate: int = 16000, channels: int = 1):33 """Record audio using sounddevice and save as 16-bit PCM WAV. Returns (filepath, duration)."""34 try:35 st.info(f"🎙️ Recording for {duration_seconds} seconds...")36 recording = sd.rec(int(duration_seconds * samplerate), samplerate=samplerate, channels=channels, dtype='int16')37 sd.wait()38 39 unique_id = uuid.uuid4()40 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")41 filename = f"voice_{timestamp}_{unique_id}.wav"42 filepath = os.path.join("voice_inputs", filename)43 # Write WAV44 sf.write(filepath, recording, samplerate, subtype='PCM_16')45 return filepath, float(duration_seconds)46 except Exception as e:47 st.error(f"Error recording audio: {str(e)}")48 return None, None49 50 def recognize_speech(self, audio, language='en-US'):51 """Convert audio to text using Google Speech Recognition (simple)"""52 try:53 text = self.recognizer.recognize_google(audio, language=language)54 return text55 except sr.UnknownValueError:56 return "Could not understand audio"57 except sr.RequestError as e:58 return f"Error with speech recognition service: {str(e)}"59 except Exception as e:60 return f"Error recognizing speech: {str(e)}"61 62 def recognize_with_details(self, audio, language='en-US'):63 """Recognize speech and return transcript, confidence, and raw result."""64 try:65 raw = self.recognizer.recognize_google(audio, language=language, show_all=True)66 transcript = ""67 confidence = None68 if isinstance(raw, dict):69 alternatives = raw.get('alternative', [])70 if alternatives:71 transcript = alternatives[0].get('transcript', '')72 confidence = alternatives[0].get('confidence')73 # Fallback to simple recognition if needed74 if not transcript:75 transcript = self.recognizer.recognize_google(audio, language=language)76 return {77 'transcript': transcript,78 'confidence': confidence,79 'raw': raw80 }81 except sr.UnknownValueError:82 return {83 'transcript': "",84 'confidence': None,85 'raw': {'error': 'unknown_value'}86 }87 except sr.RequestError as e:88 return {89 'transcript': "",90 'confidence': None,91 'raw': {'error': f'request_error: {str(e)}'}92 }93 except Exception as e:94 return {95 'transcript': "",96 'confidence': None,97 'raw': {'error': f'exception: {str(e)}'}98 }99 100 def load_audio_for_recognition(self, wav_path: str):101 """Load a WAV file into sr.AudioData for recognition."""102 try:103 with sr.AudioFile(wav_path) as source:104 audio = self.recognizer.record(source)105 return audio106 except Exception as e:107 st.error(f"Error loading audio for recognition: {str(e)}")108 return None109 110 def get_language_code(self, language_name):111 """Convert language name to speech recognition language code"""112 language_codes = {113 "English": "en-US",114 "Hindi": "hi-IN",115 "Bengali": "bn-IN",116 "Telugu": "te-IN",117 "Marathi": "mr-IN",118 "Tamil": "ta-IN",119 "Gujarati": "gu-IN",120 "Urdu": "ur-IN",121 "Kannada": "kn-IN",122 "Odia": "or-IN",123 "Malayalam": "ml-IN",124 "Punjabi": "pa-IN",125 "Assamese": "as-IN",126 "Nepali": "ne-NP",127 "Sanskrit": "sa-IN",128 }129 return language_codes.get(language_name, "en-US")130 131 132def create_voice_input_component(language="English", key_suffix=""):133 """Create a voice input component for Streamlit"""134 135 # Initialize speech recognizer136 if f'speech_recognizer_{key_suffix}' not in st.session_state:137 st.session_state[f'speech_recognizer_{key_suffix}'] = SpeechRecognizer()138 139 recognizer = st.session_state[f'speech_recognizer_{key_suffix}']140 141 # Voice input section142 st.markdown("### 🎤 Voice Input")143 144 col1, col2 = st.columns([1, 1])145 146 with col1:147 duration = st.slider("Recording duration (seconds)", min_value=3, max_value=30, value=10, key=f"duration_{key_suffix}")148 if st.button("🎤 Record", key=f"record_{key_suffix}", type="primary"):149 with st.spinner("🎧 Recording and processing..."):150 # Record to WAV using sounddevice151 audio_path, audio_duration = recognizer.record_with_sounddevice(duration_seconds=duration)152 if audio_path:153 st.session_state[f'voice_audio_path_{key_suffix}'] = audio_path154 st.session_state[f'voice_audio_duration_{key_suffix}'] = audio_duration155 # Get language code156 lang_code = recognizer.get_language_code(language)157 # Load for recognition158 audio_data = recognizer.load_audio_for_recognition(audio_path)159 if audio_data is not None:160 result = recognizer.recognize_with_details(audio_data, language=lang_code)161 transcript = result.get('transcript', '')162 confidence = result.get('confidence')163 raw = result.get('raw')164 165 st.session_state[f'voice_text_{key_suffix}'] = transcript166 st.session_state[f'voice_confidence_{key_suffix}'] = confidence167 st.session_state[f'voice_raw_{key_suffix}'] = raw168 st.session_state[f'voice_language_code_{key_suffix}'] = lang_code169 st.session_state[f'voice_provider_{key_suffix}'] = 'google_speech_recognition'170 171 if not transcript:172 st.error("❌ Could not understand audio or recognition failed")173 else:174 st.success("✅ Speech recognized successfully!")175 else:176 st.warning("⚠️ Recording failed. Please try again.")177 178 with col2:179 # Clear button180 if st.button("🗑️ Clear", key=f"clear_{key_suffix}"):181 if f'voice_text_{key_suffix}' in st.session_state:182 del st.session_state[f'voice_text_{key_suffix}']183 st.success("✅ Voice input cleared!")184 185 # Display recognized text186 if f'voice_text_{key_suffix}' in st.session_state:187 recognized_text = st.session_state[f'voice_text_{key_suffix}']188 189 if not ("Could not understand" in recognized_text or "Error" in recognized_text):190 st.markdown("### 📝 Recognized Text")191 192 # Editable text area with recognized text193 edited_text = st.text_area(194 f"Edit caption in {language}",195 value=recognized_text,196 height=150,197 key=f"edit_text_{key_suffix}",198 help=f"You can edit the recognized text or use it as is",199 )200 201 return edited_text202 else:203 st.error(f"❌ {recognized_text}")204 return ""205 206 return ""207 208 209def install_speech_dependencies():210 """Check presence of required dependencies for voice capture and recognition"""211 try:212 import sounddevice # noqa: F401213 import soundfile # noqa: F401214 import speech_recognition # noqa: F401215 return True216 except ImportError:217 return False218 219 220 221# Indian languages with their codes222INDIAN_LANGUAGES = {223 "English": "en",224 "Hindi": "hi",225 "Bengali": "bn", 226 "Telugu": "te", 227 "Marathi": "mr",228 "Tamil": "ta",229 "Gujarati": "gu",230 "Urdu": "ur",231 "Kannada": "kn",232 "Odia": "or",233 "Malayalam": "ml",234 "Punjabi": "pa",235 "Assamese": "as",236 "Maithili": "mai",237 "Sanskrit": "sa",238 "Nepali": "ne",239 "Konkani": "kok",240 "Manipuri": "mni",241 "Bodo": "brx",242 "Dogri": "doi",243 "Kashmiri": "ks",244 "Santali": "sat",245 "Sindhi": "sd"246}247 248def initialize_directories():249 """Create necessary directories if they don't exist"""250 directories = ["uploaded_images", "captioned_images", "metadata"]251 for directory in directories:252 if not os.path.exists(directory):253 os.makedirs(directory)254 255def save_image_and_metadata(uploaded_file, caption, language, metadata_extras=None):256 """Save uploaded image and its metadata. Optionally include extra metadata fields."""257 initialize_directories()258 259 # Generate unique filename260 file_extension = uploaded_file.name.split('.')[-1]261 unique_filename = f"{uuid.uuid4()}.{file_extension}"262 263 # Save image264 image_path = os.path.join("uploaded_images", unique_filename)265 with open(image_path, "wb") as f:266 f.write(uploaded_file.getbuffer())267 268 # Save metadata269 metadata = {270 "filename": unique_filename,271 "original_name": uploaded_file.name,272 "caption": caption,273 "language": language,274 "language_code": INDIAN_LANGUAGES[language],275 "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),276 "file_size": uploaded_file.size,277 "input_method": "Text",278 "audio_file": None279 }280 if metadata_extras:281 metadata.update(metadata_extras)282 283 # Load existing metadata or create new284 metadata_file = "metadata/image_metadata.csv"285 if os.path.exists(metadata_file):286 df = pd.read_csv(metadata_file)287 df = pd.concat([df, pd.DataFrame([metadata])], ignore_index=True)288 else:289 df = pd.DataFrame([metadata])290 # Ensure only one caption column and include audio_file column291 df = df.drop(columns=["final_caption"], errors="ignore")292 if "audio_file" not in df.columns:293 df["audio_file"] = None294 295 df.to_csv(metadata_file, index=False)296 297 return unique_filename298 299def get_random_image():300 """Get a random image from uploaded_images folder"""301 initialize_directories()302 303 if not os.path.exists("uploaded_images"):304 return None305 306 images = [f for f in os.listdir("uploaded_images") if f.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp'))]307 308 if not images:309 return None310 311 return random.choice(images)312 313def save_captioned_image(filename, caption, language, metadata_extras=None):314 """Save image with new caption to captioned_images folder. Optionally include extra metadata."""315 initialize_directories()316 317 # Copy image to captioned folder318 src_path = os.path.join("uploaded_images", filename)319 dst_path = os.path.join("captioned_images", filename)320 321 if os.path.exists(src_path):322 # Copy the file323 with open(src_path, "rb") as src_file:324 with open(dst_path, "wb") as dst_file:325 dst_file.write(src_file.read())326 327 # Update metadata328 metadata = {329 "filename": filename,330 "caption": caption,331 "language": language,332 "language_code": INDIAN_LANGUAGES[language],333 "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),334 "status": "captioned",335 "input_method": "Text",336 "audio_file": None337 }338 if metadata_extras:339 metadata.update(metadata_extras)340 341 # Save captioned metadata342 captioned_metadata_file = "metadata/captioned_metadata.csv"343 if os.path.exists(captioned_metadata_file):344 df = pd.read_csv(captioned_metadata_file)345 df = pd.concat([df, pd.DataFrame([metadata])], ignore_index=True)346 else:347 df = pd.DataFrame([metadata])348 # Ensure only one caption column and include audio_file column349 df = df.drop(columns=["final_caption"], errors="ignore")350 if "audio_file" not in df.columns:351 df["audio_file"] = None352 353 df.to_csv(captioned_metadata_file, index=False)354 355 return True356 return False357 358def create_dataset_zip():359 """Create a zip file containing all images and metadata"""360 zip_buffer = io.BytesIO()361 362 with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:363 # Add uploaded images364 if os.path.exists("uploaded_images"):365 for filename in os.listdir("uploaded_images"):366 file_path = os.path.join("uploaded_images", filename)367 zip_file.write(file_path, f"uploaded_images/{filename}")368 369 # Add captioned images370 if os.path.exists("captioned_images"):371 for filename in os.listdir("captioned_images"):372 file_path = os.path.join("captioned_images", filename)373 zip_file.write(file_path, f"captioned_images/{filename}")374 375 # Add metadata files376 if os.path.exists("metadata"):377 for filename in os.listdir("metadata"):378 if filename.endswith('.csv'):379 file_path = os.path.join("metadata", filename)380 zip_file.write(file_path, f"metadata/{filename}")381 382 zip_buffer.seek(0)383 return zip_buffer.getvalue()384 385def main():386 st.set_page_config(387 page_title="Image Captioning Dataset Builder - Voice Enabled",388 page_icon="🎤",389 layout="wide"390 )391 392 st.title("🎤 Image Captioning Dataset Builder - Voice Enabled")393 st.markdown("Build your multilingual image captioning dataset with voice input support for Indian languages!")394 395 # Check if speech recognition dependencies are available396 if not install_speech_dependencies():397 st.error("⚠️ Speech recognition dependencies not installed. Please install them using: pip install speechrecognition pyaudio")398 st.info("💡 For now, you can still use the text input method.")399 400 # Initialize directories401 initialize_directories()402 403 # Sidebar for navigation404 st.sidebar.title("Navigation")405 tab = st.sidebar.radio("Choose Action", [406 "Upload & Caption Images", 407 "Caption Random Images", 408 "View Dataset", 409 "Download Dataset"410 ])411 412 if tab == "Upload & Caption Images":413 st.header("📤 Upload & Caption Images")414 415 col1, col2 = st.columns([1, 1])416 417 with col1:418 # Image upload419 uploaded_file = st.file_uploader(420 "Choose an image file", 421 type=['png', 'jpg', 'jpeg', 'gif', 'bmp'],422 help="Upload an image to add to your dataset"423 )424 425 if uploaded_file is not None:426 # Display uploaded image427 image = Image.open(uploaded_file)428 st.image(image, caption="Uploaded Image", use_column_width=True)429 430 with col2:431 if uploaded_file is not None:432 # Language selection433 selected_language = st.selectbox(434 "Select Language for Caption",435 options=list(INDIAN_LANGUAGES.keys()),436 index=0,437 help="Choose the language for your image caption"438 )439 440 # Input method selection441 input_method = st.radio(442 "Choose Input Method",443 ["🎤 Voice Input", "⌨️ Text Input"],444 horizontal=True445 )446 447 caption = ""448 449 if input_method == "🎤 Voice Input":450 # Voice input component451 caption = create_voice_input_component(selected_language, "upload")452 else:453 # Traditional text input454 caption = st.text_area(455 f"Enter caption in {selected_language}",456 height=150,457 help=f"Provide a descriptive caption in {selected_language}"458 )459 460 # Save button461 if st.button("💾 Save Image & Caption", type="primary"):462 if caption and caption.strip():463 # Gather voice metadata if any464 extras = {}465 if input_method == "🎤 Voice Input":466 extras.update({467 "input_method": "Voice",468 "audio_file": st.session_state.get('voice_audio_path_upload'),469 "voice_audio_duration": st.session_state.get('voice_audio_duration_upload'),470 "voice_language_code": st.session_state.get('voice_language_code_upload'),471 "voice_confidence": st.session_state.get('voice_confidence_upload'),472 "voice_provider": st.session_state.get('voice_provider_upload'),473 })474 else:475 extras.update({"input_method": "Text"})476 477 filename = save_image_and_metadata(478 uploaded_file, caption, selected_language, metadata_extras=extras479 )480 st.success(f"✅ Image saved successfully as {filename}")481 st.success(f"📝 Caption: {caption}")482 st.success(f"🌐 Language: {selected_language}")483 st.success(f"🎯 Input Method: {input_method}")484 else:485 st.error("Please enter a caption before saving!")486 487 elif tab == "Caption Random Images":488 st.header("🎲 Caption Random Images")489 490 col1, col2 = st.columns([1, 1])491 492 with col1:493 if st.button("🎲 Get Random Image", type="primary"):494 random_image = get_random_image()495 if random_image:496 st.session_state.random_image = random_image497 else:498 st.warning("No images found! Please upload some images first.")499 500 if "random_image" in st.session_state:501 image_path = os.path.join("uploaded_images", st.session_state.random_image)502 if os.path.exists(image_path):503 image = Image.open(image_path)504 st.image(image, caption=f"Random Image: {st.session_state.random_image}", use_column_width=True)505 506 with col2:507 if "random_image" in st.session_state:508 # Language selection for captioning509 selected_language = st.selectbox(510 "Select Language for New Caption",511 options=list(INDIAN_LANGUAGES.keys()),512 index=0,513 key="caption_language"514 )515 516 # Input method selection517 input_method = st.radio(518 "Choose Input Method",519 ["🎤 Voice Input", "⌨️ Text Input"],520 horizontal=True,521 key="random_input_method"522 )523 524 new_caption = ""525 526 if input_method == "🎤 Voice Input":527 # Voice input component528 new_caption = create_voice_input_component(selected_language, "random")529 else:530 # Traditional text input531 new_caption = st.text_area(532 f"Enter new caption in {selected_language}",533 height=150,534 key="new_caption"535 )536 537 # Save caption button538 if st.button("💾 Save Caption", type="primary"):539 if new_caption and new_caption.strip():540 extras = {}541 if input_method == "🎤 Voice Input":542 extras.update({543 "input_method": "Voice",544 "audio_file": st.session_state.get('voice_audio_path_random'),545 "voice_audio_duration": st.session_state.get('voice_audio_duration_random'),546 "voice_language_code": st.session_state.get('voice_language_code_random'),547 "voice_confidence": st.session_state.get('voice_confidence_random'),548 "voice_provider": st.session_state.get('voice_provider_random'),549 })550 else:551 extras.update({"input_method": "Text"})552 553 success = save_captioned_image(554 st.session_state.random_image, 555 new_caption, 556 selected_language,557 metadata_extras=extras558 )559 if success:560 st.success("✅ Caption saved successfully!")561 st.success(f"📝 Caption: {new_caption}")562 st.success(f"🌐 Language: {selected_language}")563 st.success(f"🎯 Input Method: {input_method}")564 # Clear the random image to get a new one565 del st.session_state.random_image566 else:567 st.error("Error saving caption!")568 else:569 st.error("Please enter a caption before saving!")570 571 elif tab == "View Dataset":572 st.header("👁️ View Dataset")573 574 # Show statistics575 col1, col2, col3, col4 = st.columns(4)576 577 uploaded_count = len([f for f in os.listdir("uploaded_images") if f.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp'))]) if os.path.exists("uploaded_images") else 0578 captioned_count = len([f for f in os.listdir("captioned_images") if f.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp'))]) if os.path.exists("captioned_images") else 0579 580 with col1:581 st.metric("📤 Uploaded Images", uploaded_count)582 583 with col2:584 st.metric("📝 Captioned Images", captioned_count)585 586 with col3:587 st.metric("📊 Total Images", uploaded_count + captioned_count)588 589 with col4:590 # Count unique languages591 unique_languages = set()592 if os.path.exists("metadata/image_metadata.csv"):593 df = pd.read_csv("metadata/image_metadata.csv")594 unique_languages.update(df['language'].unique())595 if os.path.exists("metadata/captioned_metadata.csv"):596 df = pd.read_csv("metadata/captioned_metadata.csv")597 unique_languages.update(df['language'].unique())598 st.metric("🌐 Languages", len(unique_languages))599 600 # Show metadata tables601 st.subheader("📊 Dataset Metadata")602 603 tab1, tab2 = st.tabs(["Uploaded Images", "Captioned Images"])604 605 with tab1:606 if os.path.exists("metadata/image_metadata.csv"):607 df = pd.read_csv("metadata/image_metadata.csv")608 st.dataframe(df, use_container_width=True)609 else:610 st.info("No uploaded images metadata found.")611 612 with tab2:613 if os.path.exists("metadata/captioned_metadata.csv"):614 df = pd.read_csv("metadata/captioned_metadata.csv")615 st.dataframe(df, use_container_width=True)616 else:617 st.info("No captioned images metadata found.")618 619 elif tab == "Download Dataset":620 st.header("📥 Download Dataset")621 622 st.markdown("""623 Download your complete image captioning dataset including:624 - **Uploaded Images**: Original images with captions625 - **Captioned Images**: Images with additional captions 626 - **Metadata**: CSV files containing all caption and language information627 """)628 629 # Show dataset summary630 uploaded_count = len([f for f in os.listdir("uploaded_images") if f.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp'))]) if os.path.exists("uploaded_images") else 0631 captioned_count = len([f for f in os.listdir("captioned_images") if f.lower().endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp'))]) if os.path.exists("captioned_images") else 0632 633 if uploaded_count > 0 or captioned_count > 0:634 col1, col2 = st.columns([1, 1])635 636 with col1:637 st.info(f"📊 **Dataset Summary:**\n- {uploaded_count} uploaded images\n- {captioned_count} captioned images\n- {uploaded_count + captioned_count} total images")638 639 with col2:640 # Create download button641 zip_data = create_dataset_zip()642 643 st.download_button(644 label="📥 Download Complete Dataset (ZIP)",645 data=zip_data,646 file_name=f"image_dataset_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip",647 mime="application/zip",648 type="primary"649 )650 651 st.success("✅ Click the button above to download your dataset!")652 else:653 st.warning("⚠️ No images found in the dataset. Please upload some images first!")654 655 # Footer656 st.markdown("---")657 st.markdown("Built with ❤️ using Streamlit | Voice Input Support | 23+ Indian Languages")658 659if __name__ == "__main__":660 main()661 662