zivpollak/hands

1# Hand direction detection using MediaPipe2# Streamlit app for real-time hand direction detection3 4import streamlit as st5import cv26import numpy as np7import mediapipe as mp8from PIL import Image9import time10import av11from streamlit_webrtc import webrtc_streamer, VideoTransformerBase, RTCConfiguration12 13# Constants14UPSCALE = 2.0  # upsample factor for detection when using full-frame fallback15 16def add_blue_square_overlay(image):17    """Add a blue square overlay to the image that is 20% of the image size"""18    try:19        # Create a copy of the image to avoid modifying the original20        image_copy = image.copy()21        22        # Get image dimensions23        height, width = image_copy.shape[:2]24        25        # Calculate square size (20% of the smaller dimension)26        square_size = int(min(height, width) * 0.2)27        28        # Calculate position (centered on screen)29        x1 = (width - square_size) // 230        y1 = (height - square_size) // 231        x2 = x1 + square_size32        y2 = y1 + square_size33        34        # Draw square with thick border (BGR format for OpenCV)35        cv2.rectangle(image_copy, (x1, y1), (x2, y2), (255, 0, 0), 5)  # Thick border for visibility36        37        # Convert BGR to RGB for display38        image_rgb = cv2.cvtColor(image_copy, cv2.COLOR_BGR2RGB)39        40        return image_rgb, (x1, y1, x2, y2)41        42    except Exception as e:43        print(f"Error adding blue square overlay: {str(e)}")44        return image, (0, 0, image.shape[1], image.shape[0])45 46def detect_hand_landmarks(image, hands_detector, square_coords=None):47    """Detect hand landmarks and return annotated image with detection info"""48    try:49        # Convert RGB to BGR for MediaPipe50        image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)51        52        # If square coordinates are provided, crop the image to the square region53        if square_coords is not None:54            x1, y1, x2, y2 = square_coords55            # Ensure coordinates are within image bounds56            h, w = image_bgr.shape[:2]57            x1 = max(0, min(x1, w))58            y1 = max(0, min(y1, h))59            x2 = max(0, min(x2, w))60            y2 = max(0, min(y2, h))61            62            # Crop the image to the square region63            cropped_image = image_bgr[y1:y2, x1:x2]64            65            # Convert cropped image to RGB for MediaPipe66            cropped_rgb = cv2.cvtColor(cropped_image, cv2.COLOR_BGR2RGB)67            68            # Process only the cropped region69            results = hands_detector.process(cropped_rgb)70        else:71            # Process the full image72            results = hands_detector.process(image)73        74        # Check if hand was detected75        hand_detected = results and results.multi_hand_landmarks and len(results.multi_hand_landmarks) > 076        77        # Draw landmarks on the full image78        annotated_image = image_bgr.copy()79        finger_angle = None80        81        if hand_detected:82            # Initialize drawing utilities83            mp_hands = mp.solutions.hands84            mp_drawing = mp.solutions.drawing_utils85            mp_drawing_styles = mp.solutions.drawing_styles86            87            for hand_landmarks in results.multi_hand_landmarks:88                # If we cropped the image, we need to adjust landmark coordinates back to full image89                if square_coords is not None:90                    x1, y1, x2, y2 = square_coords91                    # Adjust landmark coordinates to full image space92                    adjusted_landmarks = []93                    for landmark in hand_landmarks.landmark:94                        # Scale coordinates back to full image95                        adjusted_x = x1 + landmark.x * (x2 - x1)96                        adjusted_y = y1 + landmark.y * (y2 - y1)97                        adjusted_landmarks.append(type(landmark)(x=adjusted_x, y=adjusted_y, z=landmark.z))98                    99                    # Create a new hand landmarks object with adjusted coordinates100                    adjusted_hand_landmarks = type(hand_landmarks)(landmark=adjusted_landmarks)101                    102                    # Draw landmarks and connections on full image103                    mp_drawing.draw_landmarks(104                        annotated_image,105                        adjusted_hand_landmarks,106                        mp_hands.HAND_CONNECTIONS,107                        mp_drawing_styles.get_default_hand_landmarks_style(),108                        mp_drawing_styles.get_default_hand_connections_style()109                    )110                    111                    # Draw index finger line and calculate angle with adjusted coordinates112                    finger_angle = draw_index_finger_line(annotated_image, adjusted_landmarks)113                else:114                    # Draw landmarks and connections on full image115                    mp_drawing.draw_landmarks(116                        annotated_image,117                        hand_landmarks,118                        mp_hands.HAND_CONNECTIONS,119                        mp_drawing_styles.get_default_hand_landmarks_style(),120                        mp_drawing_styles.get_default_hand_connections_style()121                    )122                    123                    # Draw index finger line and calculate angle124                    finger_angle = draw_index_finger_line(annotated_image, hand_landmarks.landmark)125        126        # Convert back to RGB127        annotated_image_rgb = cv2.cvtColor(annotated_image, cv2.COLOR_BGR2RGB)128        129        return annotated_image_rgb, hand_detected, finger_angle130            131    except Exception as e:132        print(f"Error detecting hand landmarks: {str(e)}")133        return image, False, None134 135def draw_index_finger_line(image, landmarks):136    """Draw a line connecting landmarks 5, 6, 7, 8 (index finger) and calculate angle"""137    try:138        if landmarks is None or len(landmarks) < 9:139            return None140        141        # Get image dimensions142        height, width = image.shape[:2]143        144        # Convert normalized coordinates to pixel coordinates145        points = []146        for i in [5, 6, 7, 8]:  # Index finger landmarks147            if i < len(landmarks):148                x = int(landmarks[i].x * width)149                y = int(landmarks[i].y * height)150                points.append((x, y))151        152        if len(points) < 4:153            return None154        155        # Draw the line connecting all four points156        for i in range(len(points) - 1):157            cv2.line(image, points[i], points[i + 1], (0, 255, 0), 3)158        159        # Draw circles at each landmark160        for point in points:161            cv2.circle(image, point, 3, (255, 0, 0), -1)162        163        # Calculate angle of the line from first to last point164        start_point = points[0]  # Landmark 5165        end_point = points[-1]   # Landmark 8166        167        # Calculate angle in degrees168        dx = end_point[0] - start_point[0]169        dy = end_point[1] - start_point[1]170        angle = np.degrees(np.arctan2(dy, dx))171        172        # Normalize angle to 0-360 degrees173        if angle < 0:174            angle += 360175        176        return angle177        178    except Exception as e:179        print(f"Error drawing finger line: {str(e)}")180        return None181 182def get_direction(angle):183    """Get direction based on angle"""184    if angle is None:185        return "Unknown"186    187    if (0 <= angle <= 45) or (315 <= angle <= 360):188        return "Left"189    elif 45 < angle <= 135:190        return "Down"191    elif 135 < angle <= 225:192        return "Right"193    elif 225 < angle < 315:194        return "Up"195    else:196        return "Unknown"197 198# Global hands detector199hands_detector = None200 201def initialize_hands_detector():202    """Initialize MediaPipe hands detector"""203    global hands_detector204    if hands_detector is None:205        mp_hands = mp.solutions.hands206        hands_detector = mp_hands.Hands(207            static_image_mode=False,208            max_num_hands=2,209            min_detection_confidence=0.5,210            min_tracking_confidence=0.4211        )212    return hands_detector213 214def process_frame(frame):215    """Process a single frame for live video streaming"""216    try:217        if frame is None:218            return None219        220        # Initialize hands detector221        detector = initialize_hands_detector()222        223        # Add blue square overlay224        frame_with_square, square_coords = add_blue_square_overlay(frame)225        226        # Detect hand landmarks ONLY within the blue square region227        annotated_frame, hand_detected, finger_angle = detect_hand_landmarks(frame, detector, square_coords)228        229        # Create final frame with square overlay230        x1, y1, x2, y2 = square_coords231        final_frame = annotated_frame.copy()232        cv2.rectangle(final_frame, (x1, y1), (x2, y2), (255, 0, 0), 5)233        234        # Add text overlays235        if hand_detected:236            # Add HAND text in top right237            cv2.putText(final_frame, "HAND", (final_frame.shape[1] - 150, 50), 238                       cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 3)239            240            # Add direction text under HAND241            if finger_angle is not None:242                direction = get_direction(finger_angle)243                cv2.putText(final_frame, direction, (final_frame.shape[1] - 150, 100), 244                           cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 255), 3)245        246        # Add angle text in top left247        if finger_angle is not None:248            cv2.putText(final_frame, f"Angle: {finger_angle:.1f}°", (10, 50), 249                       cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 2)250        251        return final_frame252        253    except Exception as e:254        print(f"Error processing frame: {str(e)}")255        return frame256 257# Streamlit Video Transformer Class258class HandDetectionTransformer(VideoTransformerBase):259    def __init__(self):260        self.detector = initialize_hands_detector()261    262    def recv(self, frame):263        # Convert frame to numpy array264        img = frame.to_ndarray(format="bgr24")265        266        # Process the frame267        processed_frame = process_frame(img)268        269        # Convert back to video frame format270        return av.VideoFrame.from_ndarray(processed_frame, format="bgr24")271 272# Streamlit App273st.set_page_config(274    page_title="Hand Direction Detection",275    page_icon="👉",276    layout="wide"277)278 279# Title and description280st.title("👉 Hand Direction Detection")281st.markdown("**Use your pointing finger to show the desired direction**")282 283# Sidebar with controls284with st.sidebar:285    # Detection settings286    st.header("⚙️ Settings")287    min_detection_confidence = st.slider(288        "Min Detection Confidence", 289        min_value=0.1, 290        max_value=1.0, 291        value=0.5, 292        step=0.1293    )294    min_tracking_confidence = st.slider(295        "Min Tracking Confidence", 296        min_value=0.1, 297        max_value=1.0, 298        value=0.4, 299        step=0.1300    )301 302# Main content area303col1, col2 = st.columns([2, 1])304 305with col1:306    st.header("📷 Live Camera Feed")307    308    # WebRTC streamer309    webrtc_ctx = webrtc_streamer(310        key="hand-detection",311        video_processor_factory=HandDetectionTransformer,312        rtc_configuration=RTCConfiguration(313            {"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]}314        ),315        media_stream_constraints={"video": True, "audio": False},316        async_processing=True,317    )318 319with col2:320    st.header("📊 Detection Status")321    322    # Status indicators323    if webrtc_ctx.video_transformer:324        st.success("✅ Camera Connected")325        st.info("🎯 Position your hand in the blue square")326    else:327        st.warning("⚠️ Camera not connected")328    329    # Technical details330    with st.expander("🔧 Technical Details"):331        st.markdown("**MediaPipe Hand Landmarks:**")332        st.markdown("- 21 hand landmarks per hand")333        st.markdown("- Index finger landmarks: 5, 6, 7, 8")334        st.markdown("- Angle calculated from landmark 5 to 8")335        st.markdown("**Direction Mapping:**")336        st.markdown("- 0-45° and 315-360°: Left")337        st.markdown("- 45-135°: Down")338        st.markdown("- 135-225°: Right")339        st.markdown("- 225-315°: Up")340 341# Footer342st.markdown("---")343st.markdown("Built with Streamlit and MediaPipe")344 345# Update detector settings if changed346if webrtc_ctx.video_transformer:347    webrtc_ctx.video_transformer.detector = mp.solutions.hands.Hands(348        static_image_mode=False,349        max_num_hands=2,350        min_detection_confidence=min_detection_confidence,351        min_tracking_confidence=min_tracking_confidence352    )