abhicodes/Interview-AI-Video-Processing-Model
3
1import math2import os3from io import BytesIO4import gradio as gr5import cv26from PIL import Image7import requests8from transformers import pipeline9from pydub import AudioSegment10from faster_whisper import WhisperModel11import joblib12import mediapipe as mp13import numpy as np14import pandas as pd15import moviepy.editor as mpe16import time17 18body_lang_model = joblib.load('body_language.pkl')19mp_holistic = mp.solutions.holistic20holistic = mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5)21mp_face_mesh = mp.solutions.face_mesh22face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=0.5, min_tracking_confidence=0.5)23 24theme = gr.themes.Base(25 primary_hue="cyan",26 secondary_hue="blue",27 neutral_hue="slate",28)29 30model = WhisperModel("small", device="cpu", compute_type="int8")31 32API_KEY = os.getenv('HF_API_KEY')33 34pipe1 = pipeline("image-classification", model="dima806/facial_emotions_image_detection")35pipe2 = pipeline("text-classification", model="SamLowe/roberta-base-go_emotions")36# pipe3 = pipeline("audio-classification", model="ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition")37 38# FACE_API_URL = "https://api-inference.huggingface.co/models/dima806/facial_emotions_image_detection"39# TEXT_API_URL = "https://api-inference.huggingface.co/models/SamLowe/roberta-base-go_emotions"40AUDIO_API_URL = "https://api-inference.huggingface.co/models/ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition"41headers = {"Authorization": "Bearer " + API_KEY + ""}42 43 44def extract_frames(video_path):45 clip = mpe.VideoFileClip(video_path)46 clip.write_videofile('mp4file.mp4', fps=60)47 48 cap = cv2.VideoCapture('mp4file.mp4')49 fps = int(cap.get(cv2.CAP_PROP_FPS))50 total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))51 # try:52 # while True:53 # ret, frame = cap.read()54 # if not ret:55 # break56 # total_frames += 157 # except Exception as e:58 # print("Done")59 # cap.release()60 # time.sleep(3)61 # cap = cv2.VideoCapture(video_path)62 interval = int(fps/2)63 print(interval, total_frames)64 65 images = []66 result = []67 distract_count = 068 total_count = 069 output_list = []70 71 for i in range(0, total_frames, interval):72 total_count += 173 cap.set(cv2.CAP_PROP_POS_FRAMES, i)74 ret, frame = cap.read()75 76 if ret:77 image = cv2.cvtColor(cv2.flip(frame, 1), cv2.COLOR_BGR2RGB)78 image.flags.writeable = False79 results = face_mesh.process(image)80 image.flags.writeable = True81 image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)82 83 img_h, img_w, img_c = image.shape84 face_3d = []85 face_2d = []86 87 flag = False88 89 if results.multi_face_landmarks:90 for face_landmarks in results.multi_face_landmarks:91 for idx, lm in enumerate(face_landmarks.landmark):92 if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:93 if idx == 1:94 nose_2d = (lm.x * img_w, lm.y * img_h)95 nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)96 97 x, y = int(lm.x * img_w), int(lm.y * img_h)98 face_2d.append([x, y])99 face_3d.append([x, y, lm.z]) 100 face_2d = np.array(face_2d, dtype=np.float64)101 face_3d = np.array(face_3d, dtype=np.float64)102 focal_length = 1 * img_w103 cam_matrix = np.array([ [focal_length, 0, img_h / 2],104 [0, focal_length, img_w / 2],105 [0, 0, 1]])106 dist_matrix = np.zeros((4, 1), dtype=np.float64)107 success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, dist_matrix)108 rmat, jac = cv2.Rodrigues(rot_vec)109 angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)110 x = angles[0] * 360111 y = angles[1] * 360112 z = angles[2] * 360113 114 if y < -7 or y > 7 or x < -7 or x > 7:115 flag = True116 else:117 flag = False118 119 if flag == True:120 distract_count += 1121 122 image2 = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)123 results2 = holistic.process(image2)124 125 pose = results2.pose_landmarks.landmark126 pose_row = list(np.array([[landmark.x, landmark.y, landmark.z, landmark.visibility] for landmark in pose]).flatten())127 128 face = results2.face_landmarks.landmark129 face_row = list(np.array([[landmark.x, landmark.y, landmark.z, landmark.visibility] for landmark in face]).flatten())130 131 row = pose_row+face_row132 133 X = pd.DataFrame([row])134 body_language_class = body_lang_model.predict(X)[0]135 body_language_prob = body_lang_model.predict_proba(X)[0]136 137 output_dict = {}138 for class_name, prob in zip(body_lang_model.classes_, body_language_prob):139 output_dict[class_name] = prob140 141 output_list.append(output_dict)142 143 pil_image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))144 response = pipe1(pil_image)145 146 temp = {}147 for ele in response:148 label, score = ele.values()149 temp[label] = score150 result.append(temp)151 152 images.append((cv2.cvtColor(frame, cv2.COLOR_BGR2RGB), f"Sentiments: {temp}, Distraction: {1 if flag == True else 0}"))153 154 distraction_rate = distract_count/total_count155 156 total_bad_prob = 0157 total_good_prob = 0158 159 for output_dict in output_list:160 total_bad_prob += output_dict['Bad']161 total_good_prob += output_dict['Good']162 163 num_frames = len(output_list)164 avg_bad_prob = total_bad_prob / num_frames165 avg_good_prob = total_good_prob / num_frames166 167 final_output = {'Bad': avg_bad_prob, 'Good': avg_good_prob}168 169 print("Frame extraction completed.")170 171 cap.release()172 return images, result, final_output, distraction_rate173 174 175def analyze_sentiment(text):176 response = pipe2(text)177 sentiment_results = {}178 for ele in response:179 label, score = ele.values()180 sentiment_results[label] = score181 # sentiment_list = response.json()[0]182 # sentiment_results = {results['label']: results['score'] for results in sentiment_list}183 return sentiment_results184 185 186def video_to_audio(input_video):187 188 frames_images, frames_sentiments, body_language, distraction_rate = extract_frames(input_video)189 190 cap = cv2.VideoCapture(input_video)191 fps = int(cap.get(cv2.CAP_PROP_FPS))192 audio = AudioSegment.from_file(input_video)193 audio_binary = audio.export(format="wav").read()194 audio_bytesio = BytesIO(audio_binary)195 audio_bytesio2 = BytesIO(audio_binary)196 197 segments, info = model.transcribe(audio_bytesio, beam_size=5)198 199 response = requests.post(AUDIO_API_URL, headers=headers, data=audio_bytesio2)200 # response = pipe3(audio_bytesio)201 # audio_list = list(response.json())202 # formatted_response = {results['label'] : results['score'] for results in audio_list}203 print(response.json())204 formatted_response = {}205 for ele in response.json():206 score, label = ele.values()207 formatted_response[label] = score208 209 # print("Detected language '%s' with probability %f" % (info.language, info.language_probability))210 211 transcript = ''212 audio_divide_sentiment = ''213 video_sentiment_markdown = ''214 video_sentiment_final = []215 final_output = []216 217 for segment in segments:218 transcript = transcript + segment.text + " "219 transcript_segment_sentiment = analyze_sentiment(segment.text)220 audio_divide_sentiment += "[%.2fs -> %.2fs] %s : %s`\`" % (segment.start, segment.end, segment.text, transcript_segment_sentiment)221 222 emotion_totals = {223 'admiration': 0.0,224 'amusement': 0.0,225 'angry': 0.0,226 'annoyance': 0.0,227 'approval': 0.0,228 'caring': 0.0,229 'confusion': 0.0,230 'curiosity': 0.0,231 'desire': 0.0,232 'disappointment': 0.0,233 'disapproval': 0.0,234 'disgust': 0.0,235 'embarrassment': 0.0,236 'excitement': 0.0,237 'fear': 0.0,238 'gratitude': 0.0,239 'grief': 0.0,240 'happy': 0.0,241 'love': 0.0,242 'nervousness': 0.0,243 'optimism': 0.0,244 'pride': 0.0,245 'realization': 0.0,246 'relief': 0.0,247 'remorse': 0.0,248 'sad': 0.0,249 'surprise': 0.0,250 'neutral': 0.0251 }252 253 counter = 0254 for i in range(math.ceil(segment.start), math.floor(segment.end)):255 for emotion in frames_sentiments[i].keys():256 emotion_totals[emotion] += frames_sentiments[i].get(emotion)257 counter += 1258 259 for emotion in emotion_totals:260 emotion_totals[emotion] /= counter261 262 video_sentiment_final.append(emotion_totals)263 264 video_segment_sentiment = {key: value for key, value in emotion_totals.items() if value != 0.0}265 266 video_sentiment_markdown += f"Frame {fps*math.ceil(segment.start)} - Frame {fps*math.floor(segment.end)} : {video_segment_sentiment}`\`"267 268 segment_finals = {segment.id: (segment.text, segment.start, segment.end, transcript_segment_sentiment, video_segment_sentiment)}269 final_output.append(segment_finals)270 271 total_transcript_sentiment = {key: value for key, value in analyze_sentiment(transcript).items() if value >= 0.01}272 273 emotion_finals = {274 'admiration': 0.0,275 'amusement': 0.0,276 'angry': 0.0,277 'annoyance': 0.0,278 'approval': 0.0,279 'caring': 0.0,280 'confusion': 0.0,281 'curiosity': 0.0,282 'desire': 0.0,283 'disappointment': 0.0,284 'disapproval': 0.0,285 'disgust': 0.0,286 'embarrassment': 0.0,287 'excitement': 0.0,288 'fear': 0.0,289 'gratitude': 0.0,290 'grief': 0.0,291 'happy': 0.0,292 'love': 0.0,293 'nervousness': 0.0,294 'optimism': 0.0,295 'pride': 0.0,296 'realization': 0.0,297 'relief': 0.0,298 'remorse': 0.0,299 'sad': 0.0,300 'surprise': 0.0,301 'neutral': 0.0302 }303 304 for i in range(0, video_sentiment_final.__len__()-1):305 for emotion in video_sentiment_final[i].keys():306 emotion_finals[emotion] += video_sentiment_final[i].get(emotion)307 308 for emotion in emotion_finals:309 emotion_finals[emotion] /= video_sentiment_final.__len__()310 311 emotion_finals = {key: value for key, value in emotion_finals.items() if value != 0.0}312 313 print("Processing Completed!!")314 315 payload = {316 'from': 'gradio',317 'emotions_final': emotion_finals,318 'body_language': body_language,319 'distraction_rate': distraction_rate,320 'formatted_response': formatted_response,321 'total_transcript_sentiment': total_transcript_sentiment322 }323 324 # response = requests.post('http://127.0.0.1:5000/interview', json=payload)325 326 print(payload)327 328 return str(final_output), frames_images, total_transcript_sentiment, audio_divide_sentiment, formatted_response, video_sentiment_markdown, emotion_finals, body_language, {'Distraction Rate': distraction_rate}329 330 331with gr.Blocks(theme=theme, css=".gradio-container { background: rgba(255, 255, 255, 0.2) !important; box-shadow: 0 8px 32px 0 rgba( 31, 38, 135, 0.37 ) !important; backdrop-filter: blur( 10px ) !important; -webkit-backdrop-filter: blur( 10px ) !important; border-radius: 10px !important; border: 1px solid rgba( 0, 0, 0, 0.5 ) !important;}") as Video:332 with gr.Column():333 gr.Markdown("""# Interview AI Video Processing Model""")334 with gr.Row():335 gr.Markdown("""336 ### 🤖 A cross-model ML model for Video processing in Interview AI Video Processing involves combining different machine learning models to analyze sentiments expressed in healthcare-related videos.337 - Facial Expression Recognition Model [Google/vit-base-patch16-224-in21k](https://huggingface.co/google/vit-base-patch16-224-in21k) 😊😢😰338 - Speech Recognition Model [OpenAI/Whisper](https://github.com/openai/whisper) 🗣️🎤 339 - Text Analysis Model [RoBERTa-base-go-emotions](https://huggingface.co/SamLowe/roberta-base-go_emotions) 📝📜340 - Contextual Understanding Model (Sentiment Analysis) 🔄🌐341 """)342 gr.Markdown("""### By combining the outputs of these models, the cross-model approach aims to capture a more comprehensive view of the sentiments within the interview videos. This way, candidates can gain insights into thier interview experiences and emotions, facilitating better understanding and improvements in actual interviews. """)343 344 with gr.Row():345 with gr.Column():346 input_video = gr.Video(sources=["upload", "webcam"], format='mp4')347 button = gr.Button("Process", variant="primary")348 gr.Examples(inputs=input_video, examples=[os.path.join(os.path.dirname(__file__), "test_video_1.mp4")])349 with gr.Column():350 with gr.Row():351 video_sentiment_final = gr.Label(label="Video Sentiment Score")352 speech_emotions = gr.Label(label="Audio Emotion Score")353 with gr.Row():354 overall_transcript_score = gr.Label(label="Overall Transcript Score")355 body_language = gr.Label(label="Body Language")356 distraction_rate = gr.Label(label="Distraction Rate")357 358 with gr.Column():359 frames_gallery = gr.Gallery(label="Video Frames", show_label=True, elem_id="gallery", columns=[3], rows=[1], object_fit="contain", height="auto")360 with gr.Accordion(label="JSON detailed Responses", open=False):361 json_output = gr.Textbox(label="JSON Output", info="Overall scores of the above video in segments.", show_label=True, lines=5, show_copy_button=True, interactive=False)362 audio_sentiment = gr.Textbox(label="Audio Sentiments", info="Outputs of Audio Processing from the video.", show_label=True, lines=5, show_copy_button=True, interactive=False)363 video_sentiment_markdown = gr.Textbox(label="Video Sentiments", info="Outputs of Video Frames processing from the video.", show_label=True, lines=5, show_copy_button=True, interactive=False)364 365 button.click(366 fn=video_to_audio,367 inputs=input_video,368 outputs=[json_output, frames_gallery, overall_transcript_score, audio_sentiment, speech_emotions, video_sentiment_markdown, video_sentiment_final, body_language, distraction_rate]369 )370 371Video.launch()