Piyush23890/Sign_Language_Decoder

1"""2SignBridge — Indian Sign Language Smart Communication System3============================================================4Main application entry point.5  - Flask web server6  - Real-time webcam processing via OpenCV7  - MediaPipe hand landmark extraction (126 features)8  - Motion-based switching: Random Forest (static A–Z) ↔ LSTM/ONNX (dynamic words)9  - Smart sentence builder with spacing logic10  - Live English ↔ Hindi translation (deep-translator)11  - Text-to-Speech via Web Speech API (frontend) / gTTS (backend)12  - WebSocket push updates via Flask-SocketIO13"""14 15import os16import sys17import webbrowser18from threading import Timer19 20# ── Prevent mediapipe from pulling full TensorFlow at import time ──────────────21from unittest.mock import MagicMock22_mock_tf = MagicMock()23sys.modules.setdefault('tensorflow', _mock_tf)24sys.modules.setdefault('tensorflow.tools', MagicMock())25sys.modules.setdefault('tensorflow.tools.docs', MagicMock())26 27os.environ["FLASK_SOCKETIO_ASYNC_MODE"] = "threading"28 29from flask import Flask, render_template, Response30from flask_socketio import SocketIO31 32import cv233import numpy as np34import mediapipe as mp35 36# ── Resilient MediaPipe import (handles PyInstaller path differences) ──────────37try:38    import mediapipe.solutions.hands as mp_hands_mod39    import mediapipe.solutions.drawing_utils as mp_draw_mod40except ImportError:41    try:42        import mediapipe.python.solutions.hands as mp_hands_mod43        import mediapipe.python.solutions.drawing_utils as mp_draw_mod44    except ImportError:45        mp_hands_mod = mp.solutions.hands46        mp_draw_mod  = mp.solutions.drawing_utils47 48import joblib49import time50import uuid51from collections import deque52from deep_translator import GoogleTranslator53 54# ── Flask / SocketIO ────────────────────────────────────────────────────────────55app     = Flask(__name__)56socketio = SocketIO(app, async_mode="threading", cors_allowed_origins="*")57 58 59# ── Resource path helper (works both in dev and PyInstaller .exe) ───────────────60def resource_path(relative: str) -> str:61    """Return absolute path to a bundled resource."""62    base = getattr(sys, '_MEIPASS', os.path.abspath('.'))63    return os.path.join(base, relative)64 65 66# ══════════════════════════════════════════════════════════════════════════════67#  SENTENCE BUILDER68# ══════════════════════════════════════════════════════════════════════════════69class SentenceBuilder:70    """71    Maintains the current sentence string and provides smart editing helpers.72 73    Rules74    -----75    - Dynamic words (HELLO, THANK YOU) are appended with a trailing space.76    - Static letters are appended directly; a space is added by the caller77      after a configurable no-hand pause.78    - backspace() removes the last *token* (word or letter).79    - clear() resets everything.80    """81 82    def __init__(self):83        self.sentence: str = ""84 85    # ── Public API ──────────────────────────────────────────────────────────86    def add(self, token: str) -> str:87        """Append *token* (letter or word) and return the updated sentence."""88        if token == "PAUSE":89            # Insert space between words if not already present90            if self.sentence and not self.sentence.endswith(" "):91                self.sentence += " "92        else:93            self.sentence += token94        return self.sentence95 96    def add_space(self) -> str:97        if self.sentence and not self.sentence.endswith(" "):98            self.sentence += " "99        return self.sentence100 101    def backspace(self) -> str:102        """Remove the last character, letter, or whole word."""103        if not self.sentence:104            return self.sentence105        if self.sentence.endswith(" "):106            self.sentence = self.sentence[:-1]          # remove trailing space107            return self.sentence108        parts = self.sentence.rstrip().split(" ")109        if len(parts) > 1:110            self.sentence = " ".join(parts[:-1]) + " "  # remove last word111        else:112            self.sentence = self.sentence[:-1]           # remove last char113        return self.sentence114 115    def clear(self) -> str:116        self.sentence = ""117        return self.sentence118 119    def get(self) -> str:120        return self.sentence121 122    def refined(self) -> str:123        """Return sentence capitalised and terminated with a period."""124        text = " ".join(self.sentence.split())125        if text:126            text = text.capitalize()127            if not text.endswith("."):128                text += "."129        return text130 131 132# ══════════════════════════════════════════════════════════════════════════════133#  CORE SIGN LANGUAGE SYSTEM134# ══════════════════════════════════════════════════════════════════════════════135class SignLanguageSystem:136    """137    Encapsulates all ML inference and webcam processing state.138 139    Architecture140    ------------141    Frame → MediaPipe (126 landmarks) → motion score142        ├─ low motion  → Random Forest → static letter (A–Z)143        └─ high motion → LSTM / ONNX  → dynamic word  (HELLO / THANK YOU)144    """145 146    # ── Tunable constants ──────────────────────────────────────────────────147    MOTION_THRESHOLD = 0.10   # norm(Δlandmarks) boundary between static/dynamic148    STATIC_FRAMES    = 5      # stable frames required before accepting a letter149    DYNAMIC_FRAMES   = 30     # sequence length expected by LSTM150 151    STATIC_LABELS  = [chr(i) for i in range(65, 91)]        # A–Z152    DYNAMIC_LABELS = ["HELLO", "THANK YOU"]153 154    def __init__(self):155        print("[SignBridge] Loading static model …")156        self.static_model  = joblib.load(resource_path("isl_alphabet_model.pkl"),157                                         mmap_mode=None)158        self.dynamic_model = None   # lazy-loaded on first dynamic detection159 160        # MediaPipe hands161        self.hands = mp_hands_mod.Hands(162            static_image_mode=False,163            max_num_hands=2,164            min_detection_confidence=0.70,165            min_tracking_confidence=0.70,166        )167 168        # State169        self.prev_keypoints : np.ndarray | None = None170        self.stable_count   : int  = 0171        self.static_locked  : bool = False172        self.dynamic_buf    : deque = deque(maxlen=self.DYNAMIC_FRAMES)173 174        self.sb             = SentenceBuilder()175        self.display_sign   : str  = ""176        self.can_add_space  : bool = False177        self.language       : str  = "en"178        self.translated     : str  = ""179 180        self.camera = cv2.VideoCapture(0)181        print("[SignBridge] Camera opened.")182 183    # ── Low-level helpers ──────────────────────────────────────────────────184    def _extract_keypoints(self, results) -> np.ndarray:185        """Return 126-dim vector [left_63 | right_63], zero-padded if absent."""186        left  = np.zeros(63)187        right = np.zeros(63)188        if results.multi_hand_landmarks and results.multi_handedness:189            for i, hl in enumerate(results.multi_hand_landmarks):190                label = results.multi_handedness[i].classification[0].label191                pts   = []192                for lm in hl.landmark:193                    pts.extend([lm.x, lm.y, lm.z])194                if label == "Left":195                    left  = np.array(pts)196                else:197                    right = np.array(pts)198        return np.concatenate([left, right])199 200    def _load_dynamic_model(self):201        """Lazy-load ONNX dynamic model (saves startup RAM)."""202        if self.dynamic_model is None:203            print("[SignBridge] Lazy-loading ONNX dynamic model …")204            import onnxruntime as ort205            sess = ort.InferenceSession(resource_path("dynamic_sign_model.onnx"))206            self.dynamic_model      = sess207            self.dynamic_input_name = sess.get_inputs()[0].name208            print("[SignBridge] ONNX model ready.")209 210    def _translate(self) -> str:211        """Translate the current sentence; return translated string."""212        refined = self.sb.refined()213        if self.language == "hi" and refined:214            try:215                return GoogleTranslator(source='auto', target='hindi').translate(refined)216            except Exception:217                return refined + " (Translation Error)"218        return refined219 220    def _push_update(self):221        """Emit current sign + sentence to all connected WebSocket clients."""222        socketio.emit('update_status', {223            'sign'    : self.display_sign,224            'sentence': self.translated or self.sb.get(),225        })226 227    # ── Main per-frame processing ──────────────────────────────────────────228    def process_frame(self):229        """Read one webcam frame, run inference, and return the annotated frame."""230        ok, frame = self.camera.read()231        if not ok:232            return None233 234        frame = cv2.flip(frame, 1)235        rgb   = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)236        res   = self.hands.process(rgb)237 238        # Draw landmarks239        if res.multi_hand_landmarks:240            for hl in res.multi_hand_landmarks:241                mp_draw_mod.draw_landmarks(frame, hl, mp_hands_mod.HAND_CONNECTIONS)242 243        # ── No hand detected ────────────────────────────────────────────────244        if not res.multi_hand_landmarks:245            if self.can_add_space:246                self.sb.add_space()247                self.can_add_space = False248            self._reset_state()249            self.display_sign = ""250 251        # ── Hand detected ───────────────────────────────────────────────────252        else:253            kp     = self._extract_keypoints(res)254            motion = (np.linalg.norm(kp - self.prev_keypoints)255                      if self.prev_keypoints is not None else 0.0)256            self.prev_keypoints = kp257 258            # Dynamic mode: high motion259            if motion > self.MOTION_THRESHOLD:260                self.stable_count  = 0261                self.static_locked = False262                self.dynamic_buf.append(kp)263 264                if len(self.dynamic_buf) == self.DYNAMIC_FRAMES:265                    self._load_dynamic_model()266                    X = np.array(self.dynamic_buf,267                                 dtype=np.float32).reshape(1, self.DYNAMIC_FRAMES, 126)268                    try:269                        out        = self.dynamic_model.run(270                            None, {self.dynamic_input_name: X})[0][0]271                        confidence = float(np.max(out))272                        label_idx  = int(np.argmax(out))273 274                        if confidence > 0.75:275                            word = self.DYNAMIC_LABELS[label_idx]276                            self.display_sign = word277                            self.sb.add(word + " ")278                            self.can_add_space = True279                            self.dynamic_buf.clear()280                    except Exception as exc:281                        print(f"[Dynamic inference error] {exc}")282 283            # Static mode: low motion284            else:285                self.stable_count += 1286                if self.stable_count >= self.STATIC_FRAMES and not self.static_locked:287                    X    = kp.reshape(1, -1)288                    pred = self.static_model.predict(X)[0]289                    if 0 <= pred < len(self.STATIC_LABELS):290                        letter = self.STATIC_LABELS[pred]291                        self.display_sign = letter292                        self.sb.add(letter)293                        self.can_add_space = True294                        self.static_locked = True295                        self.dynamic_buf.clear()296 297        # Update translation & push to clients298        self.translated = self._translate()299        self._push_update()300        return frame301 302    # ── Helper resets ──────────────────────────────────────────────────────303    def _reset_state(self):304        self.prev_keypoints = None305        self.stable_count   = 0306        self.static_locked  = False307 308    # ── Sentence controls ──────────────────────────────────────────────────309    def clear(self):310        self.sb.clear()311        self.translated = ""312 313    def backspace(self):314        self.sb.backspace()315        self.translated = self._translate()316 317 318# ── Global singleton ────────────────────────────────────────────────────────────319system = SignLanguageSystem()320 321 322# ══════════════════════════════════════════════════════════════════════════════323#  FLASK ROUTES324# ══════════════════════════════════════════════════════════════════════════════325@app.route('/')326def index():327    return render_template('index.html')328 329 330def _generate_frames():331    """MJPEG generator — streams annotated frames to the browser."""332    while True:333        frame = system.process_frame()334        if frame is None:335            break336        ret, buf = cv2.imencode('.jpg', frame)337        if not ret:338            continue339        yield (b'--frame\r\n'340               b'Content-Type: image/jpeg\r\n\r\n'341               + buf.tobytes()342               + b'\r\n')343        time.sleep(0.01)   # ~100 fps cap; reduces CPU load & releases GIL344 345 346@app.route('/video_feed')347def video_feed():348    return Response(_generate_frames(),349                    mimetype='multipart/x-mixed-replace; boundary=frame')350 351 352# ══════════════════════════════════════════════════════════════════════════════353#  SOCKET.IO EVENTS354# ══════════════════════════════════════════════════════════════════════════════355@socketio.on('command')356def handle_command(data):357    action = data.get('action', '')358    if action == 'clear':359        system.clear()360    elif action == 'backspace':361        system.backspace()362    # 'speak' is handled client-side via Web Speech API363 364 365@socketio.on('set_language')366def handle_set_language(data):367    system.language    = data.get('language', 'en')368    system.translated  = system._translate()369    socketio.emit('update_status', {370        'sign'    : system.display_sign,371        'sentence': system.translated,372    })373 374 375@socketio.on('translate_now')376def handle_translate_now(data):377    """Translate STT text on demand (Speech-to-Text → Hindi)."""378    text   = data.get('text', '')379    target = data.get('target', 'hi')380    if text:381        try:382            out = GoogleTranslator(source='auto', target=target).translate(text)383        except Exception:384            out = text + " (Translation Error)"385        socketio.emit('stt_translation', {'translated': out})386 387 388# ══════════════════════════════════════════════════════════════════════════════389#  ENTRY POINT390# ══════════════════════════════════════════════════════════════════════════════391def _open_browser():392    webbrowser.open_new("http://127.0.0.1:5000")393 394 395if __name__ == '__main__':396    Timer(1.5, _open_browser).start()397    # debug=False prevents paging-file / memory-mapped file issues on Windows398    socketio.run(app, debug=False, port=5000, allow_unsafe_werkzeug=True)399