thienphuc12339/Lip_Reading

1# preprocessing.py2 3import cv24import mediapipe as mp5import tensorflow as tf6 7class VideoPreprocessor:8    def __init__(self):9        self.mp_face_mesh = mp.solutions.face_mesh10        # Indices for lip landmarks11        self.UPPER_LIP_INDICES = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291]12        self.LOWER_LIP_INDICES = [146, 91, 181, 84, 17, 314, 405, 321, 375, 291]13        self.LIP_INDICES = self.UPPER_LIP_INDICES + self.LOWER_LIP_INDICES14 15    def preprocess_video(self, video_path):16        cap = cv2.VideoCapture(video_path)17        frames = []18        19        # Utilize mediapipe's GPU acceleration if available20        with self.mp_face_mesh.FaceMesh(21            static_image_mode=False,22            max_num_faces=1,23            refine_landmarks=True,24            min_detection_confidence=0.5,25            min_tracking_confidence=0.526        ) as face_mesh:27            while cap.isOpened():28                ret, frame = cap.read()29                if not ret:30                    break31                # Convert the BGR image to RGB32                rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)33 34                # Process the frame and get the facial landmarks35                results = face_mesh.process(rgb_frame)36 37                if results.multi_face_landmarks:38                    # Get the landmarks for the first face39                    face_landmarks = results.multi_face_landmarks[0]40 41                    try:42                        # Extract lip landmarks43                        lip_landmarks = [face_landmarks.landmark[i] for i in self.LIP_INDICES]44 45                        # Extract bounding box around the lips46                        h, w, _ = frame.shape47                        x_coords = [int(landmark.x * w) for landmark in lip_landmarks]48                        y_coords = [int(landmark.y * h) for landmark in lip_landmarks]49 50                        x_min, x_max = max(0, min(x_coords)), min(w, max(x_coords))51                        y_min, y_max = max(0, min(y_coords)), min(h, max(y_coords))52 53                        if x_max > x_min and y_max > y_min:54                            # Crop the lip region55                            lip_frame = frame[y_min:y_max, x_min:x_max]56 57                            # Resize to 85x85 pixels58                            lip_frame_resized = cv2.resize(lip_frame, (85, 85))59 60                            # Convert to grayscale using TensorFlow61                            lip_frame_gray = tf.image.rgb_to_grayscale(lip_frame_resized)62 63                            frames.append(lip_frame_gray)64                    except Exception as e:65                        print(f"Error processing frame: {e}")66                        continue  # Skip this frame67                else:68                    print("No face landmarks detected in frame.")69 70        cap.release()71 72        if not frames:73            print("No frames extracted during preprocessing.")74            return None  # Return None to indicate failure75 76        # Stack frames into a tensor77        frames = tf.stack(frames)78 79        # Normalize the frames80        mean = tf.math.reduce_mean(frames)81        std = tf.math.reduce_std(tf.cast(frames, tf.float32))82        normalized_frames = tf.cast((frames - mean), tf.float32) / std83 84        return normalized_frames85