shivanis14/SeniorSafetyMonitoringSystem

1import cv2, os, time, math2import numpy as np3from skimage.metrics import structural_similarity as ssim4import matplotlib.pyplot as plt5 6def compute_optical_flow(prev_gray, curr_gray):7    flow = cv2.calcOpticalFlowFarneback(prev_gray, curr_gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)8    magnitude, _ = cv2.cartToPolar(flow[..., 0], flow[..., 1])9    #print(f"DEBUG : max and min values are {np.max(magnitude)} {np.min(magnitude)}")10    return np.max(magnitude)11 12def compute_orb_distance(prev_frame, curr_frame, match_threshold = 40):13    # Initialize ORB detector14    orb = cv2.ORB_create()15 16    # Find the keypoints and descriptors with ORB17    kp1, des1 = orb.detectAndCompute(prev_frame, None)18    kp2, des2 = orb.detectAndCompute(curr_frame, None)19 20    # Create BFMatcher object21    bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)22 23    # Match descriptors24    orig_matches = bf.match(des1, des2)25 26    matches = [match for match in orig_matches if match.distance < match_threshold]27 28    # Sort them in the order of their distance (descriptor similarity)29    matches = sorted(matches, key=lambda x: x.distance)30 31    # Calculate average descriptor distance of top 10% matches32    num_matches = len(matches)  # Use 10% of matches33    if num_matches == 0:34        return 035 36    max_descriptor_distance = max(match.distance for match in matches[:num_matches])37 38    # Calculate Euclidean distances (physical movement) for top matches39    euclidean_distances = []40    for match in matches[:num_matches]:41        # Get keypoint coordinates from both frames42        pt1 = np.array(kp1[match.queryIdx].pt)  # Coordinates in prev_frame43        pt2 = np.array(kp2[match.trainIdx].pt)  # Coordinates in curr_frame44 45        # Compute Euclidean distance between matched keypoints46        euclidean_distance = np.sqrt((pt1[0] - pt2[0])**2 + (pt1[1] - pt2[1])**2)47        #print(f"DEBUG!! euclidean_distance is {euclidean_distance} between {pt1} and {pt2}")48        euclidean_distances.append(euclidean_distance)49 50    # Average Euclidean distance (keypoint movement)51    max_movement_distance = np.max(euclidean_distances)52 53    # Normalize max descriptor distance (for 256-bit ORB descriptors)54    normalized_descriptor_distance = max_descriptor_distance / 25655 56    # Return both descriptor similarity and keypoint movement57    #print(f"DEBUG!! max_descriptor_distance : {max_descriptor_distance}")58    return max_movement_distance59 60 61def compute_ssim(prev_frame, curr_frame):62    return ssim(prev_frame, curr_frame, data_range=255)63 64def compute_pixel_diff(prev_frame, curr_frame):65    diff = cv2.absdiff(prev_frame, curr_frame)66    return np.mean(diff)67 68def preprocess_frame(frame, width=640, height=360):69    target_size = (width, height)70    resized_frame = cv2.resize(frame, target_size, interpolation=cv2.INTER_AREA)  # Use INTER_AREA for shrinking71    return resized_frame72 73def smooth_curve(data, window_size=5):74    return np.convolve(data, np.ones(window_size)/window_size, mode='valid')75 76def find_timestamp_clusters(fast_motion_timestamps, min_time_gap=5):77    clusters = []  # List to hold the clusters of timestamps78    current_cluster = []  # Temporary list to hold the current cluster79 80    for i, timestamp in enumerate(fast_motion_timestamps):81        # If it's the first timestamp, start a new cluster82        if i == 0:83            current_cluster.append(timestamp)84        else:85            # Check the time difference between the current and previous timestamp86            if timestamp - fast_motion_timestamps[i-1] <= min_time_gap:87                # If the difference is less than or equal to the min_time_gap, add it to the current cluster88                current_cluster.append(timestamp)89            else:90                # If the difference is greater than min_time_gap, finish the current cluster and start a new one91                clusters.append(current_cluster)92                current_cluster = [timestamp]93    94    # Add the last cluster to the clusters list95    if current_cluster:96        clusters.append(current_cluster)97    98    return clusters99 100 101def detect_fast_motion(video_path, output_dir, end_time, start_time, window_size=3, motion_threshold=0.6, step = 2):102    cap = cv2.VideoCapture(video_path)103    fps = cap.get(cv2.CAP_PROP_FPS)104    height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)105    width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)106 107    orb_scores = []108    #optical_flow_scores = []109    ssim_scores = []110    #pixel_diff_scores = []111    timestamps = []112    frame_list = []113    114    prev_frame = None115    frame_count = 0116 117    while cap.isOpened():118        ret, orig_frame = cap.read()119        if not ret:120            break121        #print(f"DEBUG!! frame : {frame_count} time : {frame_count/fps}")122 123        if height == 360 and width == 640:124            frame = orig_frame125        else:126            frame = preprocess_frame(orig_frame, width = 640, height = 360)127 128 129        if frame_count > end_time * fps:130            break131 132        if frame_count < start_time * fps or frame_count % step != 0:133            frame_count += 1134            continue135 136        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)137        138        if prev_frame is not None:139            #optical_flow_scores.append(compute_optical_flow(prev_frame, gray))140            orb_scores.append(compute_orb_distance(prev_frame, gray))141            ssim_scores.append(compute_ssim(prev_frame, gray))142            #pixel_diff_scores.append(compute_pixel_diff(prev_frame, gray))143            #print(f"DEBUG : time : {frame_count/fps} end_time : {end_time} start_time : {start_time}")144            timestamps.append(frame_count/fps)145        else:146            #optical_flow_scores.append(0)147            orb_scores.append(0)148            ssim_scores.append(1)149            timestamps.append(start_time)150            151        frame_list.append(frame)152        prev_frame = gray153        frame_count += 1154        155        #if frame_count % 100 == 0:156        #    print(f"Processed {frame_count} frames")157    158    cap.release()159    160    new_fps = len(timestamps)/ (max(timestamps) - min(timestamps))161    print(f"fps : {fps} frame_height : {height} frame_width : {width} New fps is {new_fps}")162    # Normalize scores by image diagonal * time between frame : https://chatgpt.com/share/66f684b9-dd4c-8010-bf9c-421c3c6ef84a163 164    #optical_flow_scores = np.array(optical_flow_scores) / (np.sqrt(gray.shape[0]**2 + gray.shape[1]**2) / new_fps)165    ssim_scores = (1 - np.array(ssim_scores)) * new_fps   # Invert SSIM scores166    orb_scores = (np.array(orb_scores) * new_fps)/(np.sqrt(640**2 + 360**2))167 168    # Smooth both SSIM and ORB scores169    smoothed_ssim_scores = smooth_curve(ssim_scores, window_size=window_size)170    smoothed_orb_scores = smooth_curve(orb_scores, window_size=window_size)171 172    #pixel_diff_scores = np.array(pixel_diff_scores) / np.max(pixel_diff_scores)173    174    # Combine metrics175    combined_scores = (0.3 * orb_scores) + (0.7 * ssim_scores)176    smoothed_combined_scores = (0.3 * smoothed_orb_scores) + (0.7 * smoothed_ssim_scores)177 178    # Adjust X-axis to reflect the center of the window used for smoothing179    adjusted_timestamps = timestamps[window_size // 2 : -(window_size // 2)]180 181    # Detect fast motion using sliding window182    fast_motion_timestamps = []183    fast_motion_frames = []184    fast_motion_mags = []185 186    #for i in range(len(combined_scores) - window_size + 1):187    #    window = combined_scores[i:i + window_size]188    #    if np.mean(window) > motion_threshold:189    #        #print(f"DEBUG!! mean : {np.mean(window)} i : {i + (start_time * fps)} i+window_size : {i+window_size + (start_time * fps)} window : {window}")190    #        #fast_motion_frames.extend(range(i + int(start_time * fps), i + window_size + int(start_time * fps)))191    #        fast_motion_mags.extend(combined_scores[i:i + window_size])192    #        fast_motion_timestamps.extend(timestamps[i:i + window_size])193 194    ids = []195    for i in range(len(combined_scores)):196        if combined_scores[i] > motion_threshold:197            fast_motion_mags.append(combined_scores[i])198            fast_motion_timestamps.append(timestamps[i])199            fast_motion_frames.append(frame_list[i])200            ids.append(i)201 202    padded_fast_motion_frames = []203    padded_fast_motion_timestamps = []204    205    if len(ids) < 5 and len(ids) > 0:206        #Padding fast_motion_frames and fast_motion_timestamps207        padded_fast_motion_frames.extend(frame_list[min(ids) - 2:min(ids)])208        padded_fast_motion_timestamps.extend(timestamps[min(ids) - 2:min(ids)])209        210        padded_fast_motion_frames.extend(fast_motion_frames)211        padded_fast_motion_timestamps.extend(fast_motion_timestamps)212        213        padded_fast_motion_frames.extend(frame_list[max(ids) + 1:max(ids) + 3])214        padded_fast_motion_timestamps.extend(timestamps[max(ids) + 1:max(ids) + 3])215        print(f"padded_fast_motion_timestamps are {padded_fast_motion_timestamps}. Length of padded_fast_motion_timestamps is {len(padded_fast_motion_frames)}")216    else:217        padded_fast_motion_frames = fast_motion_frames218        padded_fast_motion_timestamps = fast_motion_timestamps219 220    # Plot results221    plt.figure(figsize=(12, 6))222    plt.plot(adjusted_timestamps, smoothed_orb_scores, label='ORB Distance')223    plt.plot(adjusted_timestamps, smoothed_ssim_scores, label='Inverted SSIM')224    #plt.plot(adjusted_timestamps, optical_flow_scores, label='Optical Flow')225    plt.plot(adjusted_timestamps, smoothed_combined_scores, label='Combined Score')226    plt.axhline(y=motion_threshold, color='r', linestyle='--', label='Threshold')227    plt.xlabel('Frame')228    plt.ylabel('Normalized Score')229    plt.title('Motion Detection Metrics')230    plt.legend()231    plt.savefig(f"{output_dir}/motion_detection_plot_smoothened_{video_path.split('/')[-1].split('.')[0]}.png")232 233        # Plot results234    plt.figure(figsize=(12, 6))235    #plt.plot(timestamps, orb_scores, label='ORB Distance')236    plt.plot(timestamps, ssim_scores, label='Inverted SSIM')237    #plt.plot(timestamps, optical_flow_scores, label='Optical Flow')238    plt.plot(timestamps, combined_scores, label='Combined Score')239    plt.axhline(y=motion_threshold, color='r', linestyle='--', label='Threshold')240    plt.xlabel('Frame')241    plt.ylabel('Normalized Score')242    plt.title('Motion Detection Metrics')243    plt.legend()244    plt.savefig(f"{output_dir}/motion_detection_plot_raw_{video_path.split('/')[-1].split('.')[0]}.png")245 246    247    # Print results248    print(f"Max motion score is {np.max(combined_scores)} and mean motion score is {np.mean(combined_scores)} from {np.min(timestamps)} to {np.max(timestamps)}")249    print(f"Detected {len(fast_motion_timestamps)} frames when step = {step}.")250    try:251        print(f"fast motion between {np.min(fast_motion_timestamps)} and {np.max(fast_motion_timestamps)}")252    except:253        pass254 255    #for i in range(len(fast_motion_timestamps)):256    #    timestamp = fast_motion_timestamps[i]257    #    mag = fast_motion_mags[i]258    #    print(f"(Time: {timestamp:.2f}s) (Magnitude : {mag:.2f})")259    260    if len(fast_motion_timestamps) == 0:261        print("FAST MOTION NOT DETECTED!")262        return [], []263    elif len(fast_motion_timestamps) > 0.5 * len(combined_scores):264        print("More than half of the video has fast motion")265        return fast_motion_timestamps, padded_fast_motion_frames266    else:267        timestamp_clusters = find_timestamp_clusters(fast_motion_timestamps, min_time_gap = 5)268        for timestamp_cluster in timestamp_clusters:269            print(f"min time : {np.min(timestamp_cluster)} max time : {np.max(timestamp_cluster)} length : {len(timestamp_cluster)}")270        return timestamp_clusters, padded_fast_motion_frames271 272 273'''274# Open the video file275video_path = "../test_videos/"276mp4_files = [f for f in os.listdir(video_path) if f.endswith('.mp4')]277output_dir = "motion_detection_results"278os.system(f"rm -rf {output_dir}")279os.system(f"mkdir {output_dir}")280end_time = 15281start_time = 0282 283for mp4_file in mp4_files:284    print(f"\nAnalyzing video {mp4_file}")285 286    if mp4_file == "8.mp4":287        end_time = 60288        start_time = 0289    elif mp4_file == "6.mp4":290        end_time = 32291        start_time = 0292    elif mp4_file == "3.mp4":293        end_time = 6.5 #To remove last few frames that are blurry294        start_time = 0295    elif mp4_file == "2.mp4":296        end_time = 182297        start_time = 140298    else:299        end_time = 15300        start_time = 0301 302    #if mp4_file != "3.mp4" and mp4_file != "5.mp4" and mp4_file != "6.mp4":303    #    continue304 305    start = time.time()306    fast_motion_timestamps = detect_fast_motion(video_path + mp4_file, output_dir, end_time, start_time, motion_threshold = 1.5)307    end = time.time()308 309    print(f"Execution time for {mp4_file} : {end - start} seconds. Duration of the video was {end_time - start_time} seconds")310'''