Masterdqqq/Facial_Expression_Recognition

1"""2File: app_utils.py3Author: Elena Ryumina and Dmitry Ryumin4Description: This module contains utility functions for facial expression recognition application.5License: MIT License6"""7 8import torch9import numpy as np10import mediapipe as mp11from PIL import Image12import cv213from pytorch_grad_cam.utils.image import show_cam_on_image14 15# Importing necessary components for the Gradio app16from app.model import pth_model_static, pth_model_dynamic, cam, pth_processing17from app.face_utils import get_box, display_info18from app.config import DICT_EMO, config_data19from app.plot import statistics_plot20 21mp_face_mesh = mp.solutions.face_mesh22 23 24def preprocess_image_and_predict(inp):25    inp = np.array(inp)26 27    if inp is None:28        return None, None, None29 30    try:31        h, w = inp.shape[:2]32    except Exception:33        return None, None, None34 35    with mp_face_mesh.FaceMesh(36        max_num_faces=1,37        refine_landmarks=False,38        min_detection_confidence=0.5,39        min_tracking_confidence=0.5,40    ) as face_mesh:41        results = face_mesh.process(inp)42        if results.multi_face_landmarks:43            for fl in results.multi_face_landmarks:44                startX, startY, endX, endY = get_box(fl, w, h)45                cur_face = inp[startY:endY, startX:endX]46                cur_face_n = pth_processing(Image.fromarray(cur_face))47                with torch.no_grad():48                    prediction = (49                        torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)50                        .detach()51                        .numpy()[0]52                    )53                confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}54                grayscale_cam = cam(input_tensor=cur_face_n)55                grayscale_cam = grayscale_cam[0, :]56                cur_face_hm = cv2.resize(cur_face,(224,224))57                cur_face_hm = np.float32(cur_face_hm) / 25558                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)59 60            return cur_face, heatmap, confidences61        62        else:63            return None, None, None64 65def preprocess_video_and_predict(video):66 67    cap = cv2.VideoCapture(video)68    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))69    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))70    fps = np.round(cap.get(cv2.CAP_PROP_FPS))71 72    path_save_video_face = 'result_face.mp4'73    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))74 75    path_save_video_hm = 'result_hm.mp4'76    vid_writer_hm = cv2.VideoWriter(path_save_video_hm, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))77 78    lstm_features = []79    count_frame = 180    count_face = 081    probs = []82    frames = []83    last_output = None84    last_heatmap = None 85    cur_face = None86 87    with mp_face_mesh.FaceMesh(88    max_num_faces=1,89    refine_landmarks=False,90    min_detection_confidence=0.5,91    min_tracking_confidence=0.5) as face_mesh:92 93        while cap.isOpened():94            _, frame = cap.read()95            if frame is None: break96 97            frame_copy = frame.copy()98            frame_copy.flags.writeable = False99            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)100            results = face_mesh.process(frame_copy)101            frame_copy.flags.writeable = True102 103            if results.multi_face_landmarks:104                for fl in results.multi_face_landmarks:105                    startX, startY, endX, endY  = get_box(fl, w, h)106                    cur_face = frame_copy[startY:endY, startX: endX]107 108                    if count_face%config_data.FRAME_DOWNSAMPLING == 0:109                        cur_face_copy = pth_processing(Image.fromarray(cur_face))110                        with torch.no_grad():111                            features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()112 113                        grayscale_cam = cam(input_tensor=cur_face_copy)114                        grayscale_cam = grayscale_cam[0, :]115                        cur_face_hm = cv2.resize(cur_face,(224,224), interpolation = cv2.INTER_AREA)116                        cur_face_hm = np.float32(cur_face_hm) / 255117                        heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=False)118                        last_heatmap = heatmap119        120                        if len(lstm_features) == 0:121                            lstm_features = [features]*10122                        else:123                            lstm_features = lstm_features[1:] + [features]124 125                        lstm_f = torch.from_numpy(np.vstack(lstm_features))126                        lstm_f = torch.unsqueeze(lstm_f, 0)127                        with torch.no_grad():128                            output = pth_model_dynamic(lstm_f).detach().numpy()129                        last_output = output130 131                        if count_face == 0:132                            count_face += 1133 134                    else:135                        if last_output is not None:136                            output = last_output137                            heatmap = last_heatmap138 139                        elif last_output is None:140                            output = np.empty((1, 7))141                            output[:] = np.nan142                            143                    probs.append(output[0])144                    frames.append(count_frame)145            else:146                if last_output is not None:147                    lstm_features = []148                    empty = np.empty((7))149                    empty[:] = np.nan150                    probs.append(empty)151                    frames.append(count_frame)                        152 153            if cur_face is not None:154                heatmap_f = display_info(heatmap, 'Frame: {}'.format(count_frame), box_scale=.3)155 156                cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)157                cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)158                cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)159                vid_writer_face.write(cur_face)160                vid_writer_hm.write(heatmap_f)161 162            count_frame += 1163            if count_face != 0:164                count_face += 1165 166        vid_writer_face.release()167        vid_writer_hm.release()168 169        stat = statistics_plot(frames, probs)170 171        if not stat:172            return None, None, None, None173        174    return video, path_save_video_face, path_save_video_hm, stat