Sparsh123d/myspace

1import cv22import mediapipe as mp3import torch4import torch.nn as nn5import numpy as np6from torchvision import transforms7from PIL import Image8import base649import io10 11# Define the CNN model with batch normalization12class ASLCNN(nn.Module):13    def __init__(self):14        super(ASLCNN, self).__init__()15        self.conv1 = nn.Conv2d(1, 32, 3, 1)16        self.bn1 = nn.BatchNorm2d(32)17        self.conv2 = nn.Conv2d(32, 64, 3, 1)18        self.bn2 = nn.BatchNorm2d(64)19        self.conv3 = nn.Conv2d(64, 128, 3, 1)20        self.bn3 = nn.BatchNorm2d(128)21        self.fc1 = nn.Linear(128 * 6 * 6, 512)22        self.fc2 = nn.Linear(512, 26)  # 26 output classes (A-Z)23 24    def forward(self, x):25        x = nn.ReLU()(self.bn1(self.conv1(x)))26        x = nn.MaxPool2d(2, 2)(x)27        x = nn.ReLU()(self.bn2(self.conv2(x)))28        x = nn.MaxPool2d(2, 2)(x)29        x = nn.ReLU()(self.bn3(self.conv3(x)))30        x = nn.MaxPool2d(2, 2)(x)31        x = torch.flatten(x, 1)32        x = nn.ReLU()(self.fc1(x))33        x = self.fc2(x)34        return x35 36# Load the model37model = ASLCNN().to(torch.device('cuda' if torch.cuda.is_available() else 'cpu'))38model.load_state_dict(torch.load('asl_cnn_model.pth', map_location=torch.device('cpu'), weights_only=True))39model.eval()40 41# MediaPipe hands setup42mp_hands = mp.solutions.hands.Hands(43    static_image_mode=True,44    max_num_hands=1,45    model_complexity=1,46    min_detection_confidence=0.5,47    min_tracking_confidence=0.548)49 50# Transformation for image preprocessing51transform = transforms.Compose([52    transforms.ToPILImage(),53    transforms.Resize((64, 64)),54    transforms.Grayscale(),55    transforms.ToTensor(),56    transforms.Normalize((0.5,), (0.5,))57])58 59def decode_image(image_data):60    """Decode base64 image to numpy array."""61    image_bytes = base64.b64decode(image_data)62    image = Image.open(io.BytesIO(image_bytes))63    return cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)64 65def preprocess_hand_image(hand_image):66    """ Convert image to grayscale and apply transformations """67    hand_image = transform(hand_image).unsqueeze(0)  # Add batch dimension68    return hand_image69 70def extract_hand_image(frame, hand_landmarks):71    """ Extract hand region from the frame """72    h, w, _ = frame.shape73    x_min = int(min(lm.x for lm in hand_landmarks.landmark) * w)74    x_max = int(max(lm.x for lm in hand_landmarks.landmark) * w)75    y_min = int(min(lm.y for lm in hand_landmarks.landmark) * h)76    y_max = int(max(lm.y for lm in hand_landmarks.landmark) * h)77    return frame[y_min:y_max, x_min:x_max]78 79def process_frame(image_data):80    global mp_hands81    try:82        image = decode_image(image_data)83        results = mp_hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))84 85        if results.multi_hand_landmarks:86            for hand_landmarks in results.multi_hand_landmarks:87                hand_img = extract_hand_image(image, hand_landmarks)88                if hand_img.size > 0:  # Ensure hand_img is not empty89                    hand_img_tensor = preprocess_hand_image(hand_img)90                    with torch.no_grad():91                        prediction = model(hand_img_tensor)92                        predicted_class = torch.argmax(prediction, dim=1).item()93                        return f"ASL: {chr(predicted_class + 65)}"94        return "No hand detected"95    except Exception as e:96        # Explicitly handle timestamp mismatch errors97        error_message = str(e)98        if "timestamp mismatch" in error_message or "SetNextTimestampBound" in error_message:99            print("Timestamp mismatch detected. Reinitializing MediaPipe Hands.")100            # Reinitialize MediaPipe Hands101            mp_hands = mp.solutions.hands.Hands(102                static_image_mode=True,103                max_num_hands=1,104                model_complexity=1,105                min_detection_confidence=0.5,106                min_tracking_confidence=0.5107            )108            return "Timestamp mismatch occurred. MediaPipe Hands reinitialized."109        else:110            print(f"Unhandled exception: {e}")111            return "Unhandled error occurred."112