Abs6187/ISL_Sign_Language_Translation
0
1"""
2ISL Sign Language Translation - TechMatrix Solvers Initiative
3Main Streamlit Application
4
5Developed by: TechMatrix Solvers Team
6- Abhay Gupta (Team Lead)
7- Kripanshu Gupta (Backend Developer)
8- Dipanshu Patel (UI/UX Designer)
9- Bhumika Patel (Deployment & Female Presenter)
10
11Institution: Shri Ram Group of Institutions
12"""
13
14import streamlit as st
15
16# Configure Streamlit page first
17st.set_page_config(
18 page_title="ISL Translation - TechMatrix Solvers",
19 page_icon="🤟",
20 layout="wide",
21 initial_sidebar_state="expanded"
22)
23
24# Show loading message
25st.write("🚀 TechMatrix Solvers ISL Translator Loading...")
26
27# Import dependencies with error handling
28try:
29 import os
30 # Set environment variables for better compatibility
31 os.environ["KERAS_BACKEND"] = "tensorflow"
32 os.environ["HF_HOME"] = "/tmp/huggingface"
33 os.environ["TRANSFORMERS_CACHE"] = "/tmp/transformers"
34
35 # Core imports
36 import numpy as np
37 import pandas as pd
38 import tempfile
39 import time
40 from PIL import Image
41 import subprocess
42 from typing import NamedTuple
43 import json
44 import shutil
45 import platform
46 import uuid
47
48 # Try OpenCV import
49 try:
50 import cv2
51 except Exception as cv_error:
52 st.warning(f"OpenCV import issue: {cv_error}")
53 cv2 = None
54
55 # Try ML library imports
56 try:
57 import keras
58 from keras.models import Sequential
59 from keras.layers import LSTM, Dense, Bidirectional, Dropout, Input, BatchNormalization
60 except Exception as keras_error:
61 st.warning(f"Keras import issue: {keras_error}")
62 keras = None
63
64 # Try video processing
65 try:
66 import ffmpeg
67 except Exception as ffmpeg_error:
68 st.warning(f"FFmpeg import issue: {ffmpeg_error}")
69 ffmpeg = None
70
71 # Try HuggingFace Hub
72 try:
73 from huggingface_hub import hf_hub_download
74 except Exception as hf_error:
75 st.warning(f"HuggingFace Hub import issue: {hf_error}")
76 hf_hub_download = None
77
78 # Try custom modules with fallback - import each module separately for better error handling
79 pose_models = None
80 expression_mapping = None
81 isl_processor = None
82 utils = None
83
84 # Try importing pose_models
85 try:
86 from pose_models import create_bodypose_model, create_handpose_model
87 pose_models = True
88 st.success("✅ Pose models imported successfully")
89 except Exception as pose_error:
90 st.warning(f"Pose models import issue: {pose_error}")
91 pose_models = None
92
93 # Try importing expression_mapping
94 try:
95 from expression_mapping import expression_mapping
96 st.success("✅ Expression mapping imported successfully")
97 except Exception as expr_error:
98 st.warning(f"Expression mapping import issue: {expr_error}")
99 # Fallback expression mapping
100 expression_mapping = {
101 'hello': 0, 'thank_you': 1, 'please': 2, 'sorry': 3, 'help': 4,
102 'good': 5, 'bad': 6, 'yes': 7, 'no': 8, 'water': 9,
103 'food': 10, 'home': 11, 'work': 12, 'school': 13, 'family': 14
104 }
105
106 # Try importing ISL processor
107 try:
108 from isl_processor import ISLTranslationModel
109 isl_processor = True
110 st.success("✅ ISL processor imported successfully")
111 except Exception as isl_error:
112 st.warning(f"ISL processor import issue: {isl_error}")
113 isl_processor = None
114
115 # Try importing pose_utils
116 try:
117 import pose_utils as utils
118 st.success("✅ Pose utils imported successfully")
119 except Exception as utils_error:
120 st.warning(f"Pose utils import issue: {utils_error}")
121 utils = None
122
123 st.success("✅ Core dependencies loaded successfully!")
124except ImportError as e:
125 st.error(f"❌ Critical import error: {e}")
126 st.error("Running in fallback mode with limited functionality.")
127
128# Ensure we have utils available globally after the main import block
129if utils is None:
130 try:
131 import pose_utils as utils
132 st.info("ℹ️ Pose utils loaded on secondary attempt")
133 except ImportError as utils_error:
134 st.error(f"❌ Failed to import pose_utils: {utils_error}")
135 utils = None
136
137# Ensure expression_mapping is available and create index-to-label mapping
138if expression_mapping is None:
139 st.warning("⚠️ Using fallback expression mapping")
140 expression_mapping = {
141 0: 'hello', 1: 'thank_you', 2: 'please', 3: 'sorry', 4: 'help',
142 5: 'good', 6: 'bad', 7: 'yes', 8: 'no', 9: 'water',
143 10: 'food', 11: 'home', 12: 'work', 13: 'school', 14: 'family'
144 }
145
146# Create index-to-label mapping function for safe access
147def get_sign_label(index):
148 """Safely get sign label from prediction index"""
149 if isinstance(expression_mapping, dict):
150 return expression_mapping.get(int(index), f'unknown_sign_{index}')
151 else:
152 return f'sign_{index}'
153
154# System information will be shown in About section
155
156
157class VideoProbeResult(NamedTuple):
158 """Structure for video probe results"""
159 return_code: int
160 json: str
161 error: str
162
163
164def probe_video_info(file_path) -> VideoProbeResult:
165 """
166 Probe video file for metadata using FFprobe
167
168 Args:
169 file_path: Path to video file
170
171 Returns:
172 VideoProbeResult containing metadata
173 """
174 command_array = [
175 "ffprobe",
176 "-v", "quiet",
177 "-print_format", "json",
178 "-show_format",
179 "-show_streams",
180 file_path
181 ]
182 result = subprocess.run(
183 command_array,
184 stdout=subprocess.PIPE,
185 stderr=subprocess.PIPE,
186 universal_newlines=True
187 )
188 return VideoProbeResult(
189 return_code=result.returncode,
190 json=result.stdout,
191 error=result.stderr
192 )
193
194
195# Define feature columns for time series processing
196body_features = [f'bodypeaks_x_{i}' for i in range(15)] + [f'bodypeaks_y_{i}' for i in range(15)]
197hand0_features = [f'hand0peaks_x_{i}' for i in range(21)] + [f'hand0peaks_y_{i}' for i in range(21)] + [f'hand0peaks_peaktxt{i}' for i in range(21)]
198hand1_features = [f'hand1peaks_x_{i}' for i in range(21)] + [f'hand1peaks_y_{i}' for i in range(21)] + [f'hand1peaks_peaktxt{i}' for i in range(21)]
199
200feature_columns_processed = body_features + hand0_features + hand1_features
201label_columns = ['Expression_encoded']
202
203
204@st.cache_resource
205def create_time_series_sequences(isl_data, feature_columns, label_columns, window_size=20):
206 """
207 Creates time series sequences from DataFrame with specified window size
208
209 Args:
210 isl_data: Input DataFrame with ISL data
211 feature_columns: List of feature column names
212 label_columns: List of label column names
213 window_size: Size of temporal window for sequence creation
214
215 Returns:
216 tuple: (X_sequences, y_sequences) for training/inference
217 """
218 if isl_data.empty:
219 return [], []
220
221 X_sequences = []
222 y_sequences = []
223
224 for group, file_df in isl_data.groupby(['Type', 'Expression_encoded', 'FileName']):
225 expr_type, expression, filename = group
226
227 # Create blank frame for padding
228 blank_frame = np.zeros((1, 156))
229
230 for idx, window_data in enumerate([file_df[i:i+window_size] for i in range(0, file_df.shape[0], 1)]):
231 if window_data.shape[0] < window_size:
232 # Pad sequence with blank frames at the beginning
233 padding_needed = window_size - window_data.shape[0]
234 padded_sequence = np.concatenate(
235 (np.repeat(blank_frame, padding_needed, axis=0),
236 window_data[feature_columns].values),
237 axis=0
238 )
239 X_sequences.append(padded_sequence)
240 y_sequences.append(expression)
241 continue
242
243 X_sequences.append(window_data[feature_columns].values)
244 y_sequences.append(expression)
245
246 return X_sequences, y_sequences
247
248
249# Global translation model variable
250translation_model = None
251
252
253@st.cache_resource
254def load_translation_model():
255 """
256 Load and configure the LSTM translation model
257
258 Returns:
259 Configured Keras Sequential model for ISL translation or None if failed
260 """
261 try:
262 if keras is None or hf_hub_download is None:
263 st.warning("Keras or HuggingFace Hub not available. Model loading skipped.")
264 return None
265
266 # Download pre-trained model file
267 model_file = hf_hub_download(
268 repo_id="sunilsarolkar/isl-translation-model",
269 filename="isl_model_final.keras"
270 )
271
272 # Try to load the complete model first
273 try:
274 model = keras.models.load_model(model_file)
275 st.success("✅ Model loaded successfully from saved file")
276 return model
277 except Exception as load_error:
278 st.warning(f"Failed to load complete model: {load_error}")
279 st.info("Attempting to build model architecture and load weights...")
280
281 # Fallback: Build model architecture and load weights
282 model = Sequential()
283 model.add(Input(shape=((20, 156))))
284 model.add(keras.layers.Masking(mask_value=0.))
285 model.add(BatchNormalization())
286 model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True)))
287
288 model.add(Dropout(0.2))
289 model.add(Bidirectional(LSTM(32, recurrent_dropout=0.2)))
290
291 model.add(keras.layers.Activation('elu'))
292 model.add(Dense(32, use_bias=False, kernel_initializer='he_normal'))
293
294 model.add(BatchNormalization())
295 model.add(Dropout(0.2))
296 model.add(keras.layers.Activation('elu'))
297 model.add(Dense(32, kernel_initializer='he_normal', use_bias=False))
298
299 model.add(BatchNormalization())
300 model.add(keras.layers.Activation('elu'))
301 model.add(Dropout(0.2))
302 # Determine number of classes - use 167 for the full dataset or fallback size
303 num_classes = len(list(expression_mapping.keys())) if expression_mapping else 167
304 model.add(Dense(num_classes, activation='softmax'))
305
306 # Try to load weights
307 model.load_weights(model_file)
308 st.success("✅ Model architecture built and weights loaded successfully")
309 return model
310
311 except Exception as e:
312 st.error(f"Failed to load translation model: {e}")
313 return None
314
315
316# Load test data
317@st.cache_data
318def load_test_data():
319 """Load test dataset and file information"""
320 testing_cleaned_path = hf_hub_download(
321 repo_id="sunilsarolkar/isl-test-data",
322 filename="testing_cleaned.csv",
323 repo_type="dataset"
324 )
325
326 test_files_path = hf_hub_download(
327 repo_id="sunilsarolkar/isl-test-data",
328 filename="test_files.csv",
329 repo_type="dataset"
330 )
331
332 testing_df = pd.read_csv(testing_cleaned_path)
333 test_files_df = pd.read_csv(test_files_path)
334
335 return testing_df, test_files_df
336
337
338# Test data will be loaded when needed
339
340
341class VideoWriter:
342 """Custom video writer using FFmpeg for better compatibility"""
343
344 def __init__(self, output_file, input_fps, input_framesize, input_pix_fmt, input_vcodec):
345 self.ff_process = (
346 ffmpeg
347 .input('pipe:',
348 format='rawvideo',
349 pix_fmt="bgr24",
350 s=f'{input_framesize[1]}x{input_framesize[0]}',
351 r=input_fps)
352 .output(output_file, pix_fmt=input_pix_fmt, vcodec=input_vcodec)
353 .overwrite_output()
354 .run_async(pipe_stdin=True)
355 )
356
357 def write_frame(self, frame):
358 """Write a single frame to the video"""
359 self.ff_process.stdin.write(frame.tobytes())
360
361 def close(self):
362 """Close the video writer"""
363 self.ff_process.stdin.close()
364 self.ff_process.wait()
365
366
367def calculate_weighted_average(numbers, weights):
368 """
369 Calculate weighted average of numbers
370
371 Args:
372 numbers: List of numbers
373 weights: List of weights
374
375 Returns:
376 float: Weighted average
377 """
378 if sum(weights) == 0:
379 return 0
380 return sum(x * y for x, y in zip(numbers, weights)) / sum(weights)
381
382
383@st.cache_data
384def resize_image(image, width=None, height=None, interpolation=cv2.INTER_AREA):
385 """
386 Resize image maintaining aspect ratio
387
388 Args:
389 image: Input image
390 width: Target width
391 height: Target height
392 interpolation: OpenCV interpolation method
393
394 Returns:
395 Resized image
396 """
397 dimensions = None
398 (h, w) = image.shape[:2]
399
400 if width is None and height is None:
401 return image
402
403 if width is None:
404 ratio = height / float(h)
405 dimensions = (int(w * ratio), height)
406 else:
407 ratio = width / float(w)
408 dimensions = (width, int(h * ratio))
409
410 resized = cv2.resize(image, dimensions, interpolation=interpolation)
411 return resized
412
413
414# Page configuration already set at the top
415
416st.title('🤟 ISL Sign Language Translation - TechMatrix Solvers Initiative')
417
418# Add custom CSS for sidebar styling
419st.markdown(
420 """
421 <style>
422 [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
423 width: 350px;
424 }
425 [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
426 width: 350px;
427 margin-left: -350px;
428 }
429
430 .team-info {
431 background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
432 color: white;
433 padding: 1rem;
434 border-radius: 0.5rem;
435 margin: 1rem 0;
436 }
437
438 .tech-matrix-header {
439 background: linear-gradient(90deg, #1e3a8a, #7c3aed);
440 color: white;
441 padding: 1rem;
442 border-radius: 0.5rem;
443 text-align: center;
444 margin-bottom: 1rem;
445 }
446 </style>
447 """,
448 unsafe_allow_html=True,
449)
450
451# Add team branding header
452st.markdown(
453 """
454 <div class="tech-matrix-header">
455 <h2>🚀 TechMatrix Solvers</h2>
456 <p>Innovating Accessible Technology Solutions</p>
457 </div>
458 """,
459 unsafe_allow_html=True
460)
461
462# Sidebar configuration
463st.sidebar.title('🤟 ISL Translation System')
464st.sidebar.subheader('Configuration')
465
466# Team information in sidebar
467st.sidebar.markdown(
468 """
469 <div class="team-info">
470 <h3>👨💻 Development Team</h3>
471 <ul>
472 <li><strong>Abhay Gupta</strong> - Team Lead</li>
473 <li><strong>Kripanshu Gupta</strong> - Backend Dev</li>
474 <li><strong>Dipanshu Patel</strong> - UI/UX Designer</li>
475 <li><strong>Bhumika Patel</strong> - Deployment</li>
476 </ul>
477 <p><em>Shri Ram Group of Institutions</em></p>
478 </div>
479 """,
480 unsafe_allow_html=True
481)
482
483# Initialize frame-wise outputs storage
484frame_predictions = {}
485
486# Application mode selection
487app_mode = st.sidebar.selectbox(
488 'Choose Application Mode',
489 ['About Project', 'Test Video Translation']
490)
491
492if app_mode == 'About Project':
493 st.markdown(
494 """
495 ## 🎯 Project Overview
496
497 Welcome to the **ISL Sign Language Translation System** developed by **TechMatrix Solvers**.
498 This cutting-edge application demonstrates real-time Indian Sign Language recognition and
499 translation using advanced deep learning techniques.
500
501 ### 🏗️ Technical Architecture
502
503 Our system combines multiple state-of-the-art technologies:
504
505 1. **Body Pose Estimation**: 25-point skeletal tracking using OpenPose
506 2. **Hand Landmark Detection**: 21-point hand keypoint identification
507 3. **Temporal Modeling**: Bidirectional LSTM networks for sequence analysis
508 4. **Real-time Processing**: Optimized inference pipeline for live translation
509 """
510 )
511
512 st.markdown(
513 """
514 ### 📊 Dataset Information
515
516 Our model is trained on the comprehensive [INCLUDE dataset](https://zenodo.org/records/4010759):
517 """
518 )
519
520 # Dataset statistics table
521 dataset_stats = {
522 "Metric": [
523 "Categories", "Total Words", "Training Videos",
524 "Avg Videos/Class", "Avg Video Length", "Resolution", "Frame Rate"
525 ],
526 "Value": [
527 "15", "263", "4,257", "16.3", "2.57s", "1920x1080", "25fps"
528 ]
529 }
530 st.table(pd.DataFrame(dataset_stats))
531
532 # Display dataset processing visualization
533 try:
534 categories_image = np.array(Image.open('original_project/categories_processed.png'))
535 st.image(categories_image, caption="📈 Processed Categories Distribution")
536 except:
537 st.info("📊 Dataset visualization images will be displayed when available")
538
539 # Model architecture information
540 st.markdown(
541 """
542 ### 🧠 Neural Network Architecture
543
544 ```python
545 # TechMatrix Solvers LSTM Translation Model
546 model = Sequential([
547 Input(shape=(20, 156)), # 20-frame temporal window
548 Masking(mask_value=0.),
549 BatchNormalization(),
550 Bidirectional(LSTM(32, recurrent_dropout=0.2, return_sequences=True)),
551 Dropout(0.2),
552 Bidirectional(LSTM(32, recurrent_dropout=0.2)),
553 Dense(32, activation='elu'),
554 BatchNormalization(),
555 Dropout(0.2),
556 Dense(len(expression_mapping), activation='softmax')
557 ])
558 ```
559
560 **Model Statistics:**
561 - Total Parameters: 82,679 (322.96 KB)
562 - Trainable Parameters: 82,239 (321.25 KB)
563 - Input Features: 156-dimensional vectors
564 - Temporal Window: 20 frames
565 """
566 )
567
568 # Technology stack
569 col1, col2 = st.columns(2)
570
571 with col1:
572 st.markdown(
573 """
574 ### 🛠️ Technology Stack
575
576 **Frontend & UI:**
577 - Streamlit (Interactive Web App)
578 - Custom CSS Styling
579 - Responsive Design
580
581 **Deep Learning:**
582 - Keras/TensorFlow Backend
583 - PyTorch Integration
584 - LSTM Networks
585 - OpenPose Models
586 """
587 )
588
589 with col2:
590 st.markdown(
591 """
592 ### 📱 Key Features
593
594 **Real-time Processing:**
595 - Live video analysis
596 - Pose keypoint extraction
597 - Temporal sequence modeling
598 - Confidence scoring
599
600 **User Experience:**
601 - Intuitive interface
602 - Visual feedback
603 - Progress tracking
604 - Result visualization
605 """
606 )
607
608 # System Information
609 st.markdown("### 🔧 System Information")
610 col1, col2 = st.columns(2)
611
612 with col1:
613 st.write(f"**Python Version:** {platform.python_version()}")
614 st.write(f"**FFmpeg:** {shutil.which('ffmpeg') or 'Not found'}")
615 st.write(f"**FFprobe:** {shutil.which('ffprobe') or 'Not found'}")
616
617 with col2:
618 try:
619 st.write(f"**OpenCV Version:** {cv2.__version__}")
620 except:
621 st.write("**OpenCV:** Not available")
622 try:
623 import torch
624 st.write(f"**PyTorch:** {torch.__version__}")
625 st.write(f"**Keras:** {keras.__version__}")
626 except:
627 st.write("**PyTorch/Keras:** Not available")
628
629 # Team contact information
630 st.markdown(
631 """
632 ### 📞 Contact Information
633
634 **TechMatrix Solvers Team:**
635
636 | Name | Role | Email | Phone |
637 |------|------|-------|---------|
638 | **Abhay Gupta** | Team Lead | contact2abhaygupta6187@gmail.com | 8115814535 |
639 | **Kripanshu Gupta** | Backend Developer | guptakripanshu83@gmail.com | 7067058400 |
640 | **Dipanshu Patel** | UI/UX Designer | dipanshupatel43@gmail.com | 9294526404 |
641 | **Bhumika Patel** | Deployment & Presenter | bp7249951@gmail.com | 9302271422 |
642
643 **Institution:** Shri Ram Group of Institutions
644
645 ### 📚 Documentation
646
647 For detailed technical documentation and implementation details, please refer to our
648 [comprehensive documentation](https://docs.google.com/document/d/1mzr2KGHRJT5heUjFF20NQ3Gb89urpjZJ/edit?usp=sharing).
649
650 ---
651
652 **© 2024 TechMatrix Solvers - Innovating Accessible Technology Solutions**
653 """
654 )
655
656elif app_mode == 'Test Video Translation':
657 # Video selection interface
658 st.markdown("## 🎥 Test Video Translation")
659
660 # Load test data dynamically
661 with st.spinner("Loading test data..."):
662 try:
663 testing_df, test_files_df = load_test_data()
664 st.success("✅ Test data loaded successfully!")
665 except Exception as e:
666 st.error(f"❌ Failed to load test data: {e}")
667 st.stop()
668
669 category = st.sidebar.selectbox(
670 'Choose Category',
671 np.sort(test_files_df['Category'].unique(), axis=-1, kind='mergesort')
672 )
673
674 # Filter by category
675 category_mask = (test_files_df['Category'] == category)
676 test_files_category = test_files_df[category_mask]
677
678 class_name = st.sidebar.selectbox(
679 'Choose Class',
680 np.sort(test_files_category['Class'].unique(), axis=-1, kind='mergesort')
681 )
682
683 # Filter by class
684 class_mask = (test_files_df['Class'] == class_name)
685 filename = st.sidebar.selectbox(
686 'Choose File',
687 np.sort(test_files_category[class_mask]['Filename'].unique(), axis=-1, kind='mergesort')
688 )
689
690 # Display selection info
691 st.info(f"📂 Selected: {category} → {class_name} → {filename}")
692
693 if st.sidebar.button("🚀 Start Translation", type="primary"):
694 # Filter test data for selected video
695 data_mask = ((testing_df['FileName'] == filename) &
696 (testing_df['Type'] == category) &
697 (testing_df['Expression'] == class_name))
698
699 window_size = 20
700 current_test_data = testing_df[data_mask]
701
702 if current_test_data.empty:
703 st.error(f"⚠️ No matching data found for: {filename} | {category} | {class_name}")
704 st.stop()
705 else:
706 st.success(f"✅ Loaded {current_test_data.shape[0]} frames for processing")
707
708 # Create time series data
709 X_test_processed, y_test_processed = create_time_series_sequences(
710 current_test_data, feature_columns_processed, label_columns, window_size=window_size
711 )
712 X_test_processed = np.array(X_test_processed)
713
714 # Configure Streamlit display options
715 st.set_option('deprecation.showfileUploaderEncoding', False)
716
717 st.sidebar.markdown('---')
718 st.markdown(
719 """
720 <style>
721 [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
722 width: 400px;
723 }
724 [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
725 width: 400px;
726 margin-left: -400px;
727 }
728 </style>
729 """,
730 unsafe_allow_html=True,
731 )
732
733 st.sidebar.markdown('---')
734 st.markdown('## 📊 Translation Results')
735
736 # Progress tracking container
737 progress_container = st.empty()
738
739 with progress_container.container():
740 progress_df = pd.DataFrame([['--', '--']],
741 columns=['Frames Processed', 'Detected Sign'])
742 progress_table = st.table(progress_df)
743
744 # Video display container
745 video_display = st.empty()
746 st.markdown("<hr/>", unsafe_allow_html=True)
747 frame_display = st.empty()
748
749 # Download test video
750 video_file_path = hf_hub_download(
751 repo_id="sunilsarolkar/isl-test-data",
752 filename=f'test/{category}/{class_name}/{filename}',
753 repo_type="dataset"
754 )
755
756 if not os.path.exists(video_file_path):
757 st.error(f"⚠️ Video file not found: {video_file_path}")
758 st.stop()
759
760 # Initialize video capture
761 video_capture = cv2.VideoCapture(video_file_path)
762
763 # Get video metadata
764 probe_result = probe_video_info(video_file_path)
765 video_info = json.loads(probe_result.json)
766 video_stream = [stream for stream in video_info["streams"] if stream["codec_type"] == "video"][0]
767
768 input_fps = video_stream["avg_frame_rate"]
769 input_pix_fmt = video_stream["pix_fmt"]
770 input_vcodec = video_stream["codec_name"]
771 format_name = video_info["format"]["format_name"].split(",")[0]
772
773 # Video properties
774 width = int(video_capture.get(cv2.CAP_PROP_FRAME_WIDTH))
775 height = int(video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
776 fps_input = int(video_capture.get(cv2.CAP_PROP_FPS))
777
778 # Processing variables
779 total_frames = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
780 frame_buffer = []
781
782 # Output video configuration
783 output_file = f"/tmp/techmatrix_output_{uuid.uuid4().hex}.{format_name}"
784 video_writer = None
785 weighted_predictions = {}
786 frame_predictions = {} # Reset for each video session
787
788 frame_idx = 0
789
790 try:
791 # Process each frame
792 for _, frame_data in current_test_data.iterrows():
793 if not video_capture.isOpened():
794 st.error(f"❌ Could not open video: {video_file_path}")
795 break
796
797 if video_capture.isOpened():
798 ret, frame = video_capture.read()
799
800 if len(frame_buffer) < window_size:
801 # Initial frames - build up buffer
802 if utils is not None:
803 visualization_canvas = utils.render_stick_model(
804 frame,
805 eval(frame_data['bodypose_circles']),
806 eval(frame_data['bodypose_sticks']),
807 eval(frame_data['handpose_edges']),
808 eval(frame_data['handpose_peaks'])
809 )
810 else:
811 visualization_canvas = frame # Use original frame if utils not available
812
813 # Add prediction plots
814 if utils is not None:
815 canvas_with_predictions = utils.create_bar_plot_visualization(
816 visualization_canvas, {},
817 f'Building Buffer - Frame {frame_idx + 1} [No Predictions Yet]',
818 visualization_canvas
819 )
820 canvas_with_predictions = utils.create_bar_plot_visualization(
821 canvas_with_predictions, weighted_predictions,
822 f'Weighted Average - Frame {frame_idx + 1} [No Predictions Yet]',
823 visualization_canvas
824 )
825 canvas_with_predictions = utils.add_bottom_padding(
826 canvas_with_predictions, (255, 255, 255), 100
827 )
828 else:
829 canvas_with_predictions = visualization_canvas # Use base canvas if utils not available
830
831 # Initialize video writer
832 if video_writer is None:
833 input_framesize = canvas_with_predictions.shape[:2]
834 video_writer = VideoWriter(output_file, input_fps, input_framesize,
835 input_pix_fmt, input_vcodec)
836
837 video_writer.write_frame(canvas_with_predictions)
838
839 # Update progress display
840 with progress_container.container():
841 progress_df = pd.DataFrame(
842 [[f'{frame_idx + 1}/{current_test_data.shape[0]}',
843 '<Building 20-frame buffer>']],
844 columns=['Frames Processed', 'Detected Sign']
845 )
846 progress_table = st.table(progress_df)
847
848 frame_buffer.append(frame)
849
850 # Display current frame
851 with video_display.container():
852 st.image(canvas_with_predictions, channels='BGR', use_column_width=True)
853 else:
854 # Process with full buffer - make predictions
855 frame_buffer[:-1] = frame_buffer[1:]
856 frame_buffer[-1] = frame
857
858 # Load translation model
859 translation_model = load_translation_model()
860
861 # Check if model loaded successfully
862 sequence_idx = frame_idx - 20 # Define sequence_idx for both cases
863 if translation_model is None:
864 st.error("❌ Translation model failed to load. Cannot make predictions.")
865 # Use dummy predictions to keep the visualization working
866 current_predictions = {"model_not_available": 0.0}
867 top_3_signs = ["model_not_available"]
868 top_3_probabilities = [0.0]
869 else:
870 # Make prediction on current window
871 prediction_output = translation_model(
872 X_test_processed[sequence_idx].reshape(
873 1, X_test_processed[sequence_idx].shape[0],
874 X_test_processed[sequence_idx].shape[1]
875 )
876 )
877
878 # Handle both PyTorch and Keras/TensorFlow models
879 try:
880 # Try PyTorch tensor operations first
881 prediction_output = prediction_output[0].cpu().detach().numpy()
882 except AttributeError:
883 # If it's a Keras model, it already returns NumPy arrays
884 prediction_output = prediction_output[0]
885
886 # Get top predictions
887 top_prediction_idx = np.argmax(prediction_output)
888 top_3_indices = prediction_output.argsort()[-3:][::-1]
889 top_3_signs = [get_sign_label(i) for i in top_3_indices]
890 top_3_probabilities = prediction_output[top_3_indices]
891
892 # Current frame predictions
893 current_predictions = {}
894 for sign, prob in zip(top_3_signs, top_3_probabilities):
895 current_predictions[sign] = prob
896
897 # Update frame-wise predictions for weighted average
898 for sign, prob in zip(top_3_signs, top_3_probabilities):
899 if sign not in frame_predictions:
900 frame_predictions[sign] = []
901 frame_predictions[sign].append(prob)
902
903 # Calculate weighted averages
904 for sign in frame_predictions:
905 sign_predictions = frame_predictions[sign]
906 sign_weights = [len(sign_predictions) for _ in range(len(sign_predictions))]
907 weighted_predictions[sign] = calculate_weighted_average(
908 sign_predictions, sign_weights
909 )
910
911 # Sort predictions by confidence
912 sorted_predictions = dict(
913 sorted(weighted_predictions.items(), key=lambda item: item[1], reverse=True)
914 )
915
916 # Create visualization
917 if utils is not None:
918 visualization_canvas = utils.render_stick_model(
919 frame,
920 eval(frame_data['bodypose_circles']),
921 eval(frame_data['bodypose_sticks']),
922 eval(frame_data['handpose_edges']),
923 eval(frame_data['handpose_peaks'])
924 )
925 else:
926 visualization_canvas = frame # Use original frame if utils not available
927
928 # Add prediction visualizations
929 if utils is not None:
930 canvas_with_predictions = utils.create_bar_plot_visualization(
931 visualization_canvas, current_predictions,
932 f'Current Window Prediction (Frames {sequence_idx + 1}-{frame_idx + 1})',
933 visualization_canvas
934 )
935 canvas_with_predictions = utils.create_bar_plot_visualization(
936 canvas_with_predictions, weighted_predictions,
937 f'Cumulative Weighted Average - Frame {frame_idx + 1}',
938 visualization_canvas
939 )
940 canvas_with_predictions = utils.add_bottom_padding(
941 canvas_with_predictions, (255, 255, 255), 100
942 )
943 else:
944 canvas_with_predictions = visualization_canvas # Use base canvas if utils not available
945
946 video_writer.write_frame(canvas_with_predictions)
947
948 # Get best prediction for display
949 if weighted_predictions:
950 best_sign = max(weighted_predictions, key=weighted_predictions.get)
951 best_confidence = weighted_predictions[best_sign]
952 else:
953 best_sign = "no_predictions"
954 best_confidence = 0.0
955
956 # Update progress display
957 with progress_container.container():
958 progress_df = pd.DataFrame(
959 [[f'{frame_idx + 1}/{current_test_data.shape[0]}',
960 f'{best_sign} ({best_confidence * 100:.2f}%)']],
961 columns=['Frames Processed', 'Detected Sign']
962 )
963 progress_table = st.table(progress_df)
964
965 # Display current frame
966 with video_display.container():
967 st.image(canvas_with_predictions, channels='BGR', use_column_width=True)
968
969 frame_idx += 1
970
971 # Finalize video processing
972 st.success("✅ Video processing completed!")
973
974 with video_display.container():
975 if video_writer is not None:
976 video_writer.close()
977 with open(output_file, 'rb') as video_file:
978 output_video_bytes = video_file.read()
979 st.video(output_video_bytes)
980 st.info(f"💾 Processed video saved: {output_file}")
981 else:
982 st.warning("⚠️ No video output generated")
983
984 finally:
985 # Clean up resources
986 if 'video_capture' in locals() and video_capture is not None:
987 video_capture.release()
988 if 'video_writer' in locals() and video_writer is not None:
989 video_writer.close()
990 # Note: cv2.destroyAllWindows() removed for headless compatibility
991
992# Footer
993st.markdown(
994 """
995 ---
996 <div style="text-align: center; color: #666;">
997 <p><strong>TechMatrix Solvers</strong> | Shri Ram Group of Institutions</p>
998 <p>Innovating Accessible Technology Solutions for Everyone 🚀</p>
999 </div>
1000 """,
1001 unsafe_allow_html=True
1002)