project-sign-language/Sign_language

1import cv22import json3import numpy as np4import pandas as pd5import time6 7 8def draw_hands_connections(frame, hand_landmarks):9    '''10    Draw white lines on the given frame between relevant hand keypoints.11 12    Parameters13    ----------14    frame: numpy array15        The frame on which we want to draw.16    hand_landmarks: dict17        Dictionary mapping keypoint IDs (integers) to hand landmarks 18        (lists of two floats corresponding to the coordinates) for both hands.19 20    Returns21    -------22    frame: numpy array23        The frame with the newly drawn hand connections.24    '''25 26    # ---- Define hand_connections between keypoints to draw27    #28    hand_connections = [[0, 1], [1, 2], [2, 3], [3, 4],29                        [5, 6], [6, 7], [7, 8],30                        [9, 10], [10, 11], [11, 12],31                        [13, 14], [14, 15], [15, 16],32                        [17, 18], [18, 19], [19, 20]] #[5, 2], [0, 17]]33    34    # ---- loop to draw left hand connections35    #36    for connection in hand_connections:37        landmark_start = hand_landmarks['left_hand'].get(str(connection[0]))38        landmark_end = hand_landmarks['left_hand'].get(str(connection[1]))39        cv2.line(frame, landmark_start, landmark_end, (255, 255, 255), 2)40    41    # ---- loop to to draw right hand connections42    #43    for connection in hand_connections:44        landmark_start = hand_landmarks['right_hand'].get(str(connection[0]))45        landmark_end = hand_landmarks['right_hand'].get(str(connection[1]))46        cv2.line(frame, landmark_start, landmark_end, (255, 255, 255), 2)47    48    return frame49 50def draw_pose_connections(frame, pose_landmarks):51    '''52    Draw white lines on the given frame between relevant posture keypoints.53 54    Parameters55    ----------56    frame: numpy array57        The frame on which we want to draw.58    pose_landmarks: dict59        Dictionary mapping keypoint IDs (integers) to posture landmarks 60        (lists of two floats corresponding to the coordinates).61 62    Returns63    -------64    frame: numpy array65        The frame with the newly drawn posture connections.66    '''67 68    # ---- define posture connections between keypoints to draw69    #70    pose_connections = [[11, 12], [11, 13], [12, 14], [13, 15], [14, 16]]71 72    # ---- loop to to draw posture connections73    #74    for connection in pose_connections:75        landmark_start = pose_landmarks.get(str(connection[0]))76        landmark_end = pose_landmarks.get(str(connection[1]))77        cv2.line(frame, landmark_start, landmark_end, (255, 255, 255), 2)78 79    return frame80 81def draw_face_connections(frame, face_landmarks):82    '''83    Draw white lines on the given frame between relevant face keypoints.84 85    Parameters86    ----------87    frame: numpy array88        The frame on which we want to draw.89    face_landmarks: dict90        Dictionary mapping keypoint IDs (integers) to face landmarks 91        (lists of two floats corresponding to the coordinates).92 93    Returns94    -------95    frame: numpy array96        The frame with the newly drawn face connections.97    '''98    # ---- define pose connections99    #100    connections_dict = {'lipsUpperInner_connections' : [78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308],\101                    'lipsLowerInner_connections' : [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308],\102                    'rightEyeUpper0_connections': [246, 161, 160, 159, 158, 157, 173],\103                    'rightEyeLower0' : [33, 7, 163, 144, 145, 153, 154, 155, 133],\104                    'rightEyebrowLower' : [35, 124, 46, 53, 52, 65],\105                    'leftEyeUpper0' : [466, 388, 387, 386, 385, 384, 398],\106                    'leftEyeLower0' : [263, 249, 390, 373, 374, 380, 381, 382, 362],\107                    'leftEyebrowLower' : [265, 353, 276, 283, 282, 295],\108                    'noseTip_midwayBetweenEye' :  [1, 168],\109                    'noseTip_noseRightCorner' : [1, 98],\110                    'noseTip_LeftCorner' : [1, 327]\111                    }112 113    # ---- loop to to draw face connections114    #115    for keypoints_list in connections_dict.values():116        for index in range(len(keypoints_list)):117            if index + 1 < len(keypoints_list):118                landmark_start = face_landmarks.get(str(keypoints_list[index]))119                landmark_end = face_landmarks.get(str(keypoints_list[index+1]))120                cv2.line(frame, landmark_start, landmark_end, (255, 255, 255), 1)121    return frame122 123def resize_landmarks(landmarks, resize_rate_width, resize_rate_height):124    '''125    Resize landmark coordinates by applying specific scaling factors 126    to both the width and height of the frame.127 128    Parameters129    ----------130    landmarks: dict131        Dictionary mapping keypoint IDs (integers) to landmarks132        (lists of two floats corresponding to the coordinates).133    resize_rate_width: float134        Scaling factor applied to the x-coordinate (width).135    resize_rate_height: float136        Scaling factor applied to the y-coordinate (height).137 138    Returns139    -------140    landmarks: dict141        Dictionary mapping keypoint IDs (integers) to the newly resized landmarks142        (lists of two integers corresponding to the coordinates).143    '''144 145    for keypoint in landmarks.keys():146        landmark_x, landmark_y = landmarks[keypoint]147        landmarks[keypoint] = [int(resize_rate_width * landmark_x), int(resize_rate_height*landmark_y)]148 149    return landmarks150 151def generate_video(gloss_list, dataset, vocabulary_list):152    '''153    Generate a video stream from a list of glosses.154 155    Parameters156    ----------157    gloss_list: list of str158        List of glosses from which the signing video will be generated.159    dataset: pandas.DataFrame160        Dataset containing information about each gloss, including paths to landmark data.161    vocabulary_list: list of str162        List of tokens that have associated landmarks collected.163 164    Yields165    ------166    frame: bytes167        JPEG-encoded frame for streaming.168    '''169    # ---- Fix size of the frame to the most common size of video we have in the dataset170    # (corresponding to signer ID 11 who has the maximum number of videos).171    #172    FIXED_WIDTH,  FIXED_HEIGHT = 576, 384173 174    # ---- Fix the Frames Per Second (FPS) to match the videos collected in the dataset.175    #176    FPS = 25177 178    # ---- Define carachteristics for text display.179    #180    font = cv2.FONT_HERSHEY_SIMPLEX181    font_scale = 1182    font_color = (0, 255, 0)183    thickness = 2184    line_type = cv2.LINE_AA185 186    # ---- Loop over each gloss187    #188    for gloss in gloss_list:189        # ---- Skip if gloss not in the vocabulary_list.190        #191        if not check_gloss_in_vocabulary(gloss, vocabulary_list):192            continue193 194        # ---- Get landmarks of all the frame in the dataset corresponding to the appropriate gloss.195        #196        video_id = select_video_id_from_gloss(gloss, dataset)197        video_landmarks_path = dataset.loc[dataset['video_id'] == video_id, 'video_landmarks_path'].values[0]198        with open(video_landmarks_path, 'r') as f:199            video_landmarks = json.load(f)200        width = video_landmarks[-1].get('width')201        height = video_landmarks[-1].get('height')202 203        # ---- Calculate resize rate for future landmark rescaling.204        #205        resize_rate_width, resize_rate_height  = FIXED_WIDTH / width, FIXED_HEIGHT/height206 207        # ---- Loop over each frame208        #209        for frame_landmarks in video_landmarks[:-1]:210            # ---- Initialize blank image and get all landmarks of the given frame.211            #212            blank_image = np.zeros((FIXED_HEIGHT, FIXED_WIDTH, 3), dtype=np.uint8)213            frame_hands_landmarks = frame_landmarks['hands_landmarks']214            frame_pose_landmarks = frame_landmarks['pose_landmarks']215            frame_face_landmarks = frame_landmarks['face_landmarks']216 217            # ---- Resize landmarks.218            #219            frame_hands_landmarks_rs = {220                            'left_hand': resize_landmarks(frame_hands_landmarks['left_hand'], resize_rate_width, resize_rate_height),221                            'right_hand': resize_landmarks(frame_hands_landmarks['right_hand'], resize_rate_width, resize_rate_height)222                                        }223            frame_pose_landmarks_rs = resize_landmarks(frame_pose_landmarks, resize_rate_width, resize_rate_height)224            frame_face_landmarks_rs = resize_landmarks(frame_face_landmarks, resize_rate_width, resize_rate_height)225            226            # ---- Draw relevant connections between keypoints on the frame.227            #228            draw_hands_connections(blank_image, frame_hands_landmarks_rs)229            draw_pose_connections(blank_image, frame_pose_landmarks_rs)230            draw_face_connections(blank_image, frame_face_landmarks_rs)231 232            # ---- Display text corresponding to the gloss on the frame.233            #234            text_size, _ = cv2.getTextSize(gloss, font, font_scale, thickness)235            text_x = (FIXED_WIDTH - text_size[0]) // 2236            text_y = FIXED_HEIGHT - 10237            cv2.putText(blank_image, gloss, (text_x, text_y), font, font_scale, font_color, thickness, line_type)238            239             # ---- JPEG-encode the frame for streaming.240            #241            _, buffer = cv2.imencode('.jpg', blank_image)242            frame = buffer.tobytes()243 244            yield (b'--frame\r\n'245                   b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')246 247            time.sleep(1 / FPS)248 249 250def load_data(dataset_path='enhanced_dataset'):251    '''252    Load the dataset that contains all information about glosses.253 254    Parameters255    ----------256    dataset_path: str257        Local path to the dataset.258 259    Returns260    -------261    data_df: pandas.DataFrame262        DataFrame containing the dataset with information about each gloss.263    vocabulary_list: list of str264        List of glosses (tokens) that have associated landmarks collected.265    '''266 267    filepath = dataset_path268    data_df = pd.read_csv(filepath, dtype={'video_id': str})269    vocabulary_list = data_df['gloss'].tolist()270 271    return data_df, vocabulary_list272 273 274def check_gloss_in_vocabulary(gloss, vocabulary_list):275    '''276    Check if the given gloss is in the vocabulary list.277 278    Parameters279    ----------280    gloss: str281        The gloss to check.282    vocabulary_list: list of str283        List of glosses (tokens) that have associated landmarks collected.284 285    Returns286    -------287    bool288        True if the gloss is in the vocabulary list, False otherwise.289    '''290 291    return gloss in vocabulary_list292 293 294def select_video_id_from_gloss(gloss, dataset):295    '''296    Selects a video ID corresponding to the given gloss from the dataset.297 298    Parameters299    ----------300    gloss : str301        The gloss for which to retrieve the video ID.302    dataset : pandas.DataFrame303        A DataFrame containing information about each gloss, including 'signer_id', 'gloss', and 'video_id'.304 305    Returns306    -------307    int308        The video ID corresponding to the given gloss. If the gloss is found for 'signer_id' 11, the video ID for that signer is returned; otherwise, the video ID for the gloss from the entire dataset is returned.309    '''310    # ---- Choose preferentialy ID 11 because this signer with this ID signed the more video311    #312    filtered_data_id_11 = dataset.loc[dataset['signer_id'] == 11]313 314    if gloss in filtered_data_id_11['gloss'].tolist():315        video_id = filtered_data_id_11.loc[filtered_data_id_11['gloss'] == gloss, 'video_id'].values316    else:317        video_id = dataset.loc[dataset['gloss'] == gloss, 'video_id'].values318 319    return video_id[0]