OpenCVUniversity/Mouse-Control-using-OpenCV

1import streamlit as st2from streamlit_webrtc import webrtc_streamer, VideoProcessorBase3import av4import cv25import mediapipe as mp6import numpy as np7import pyautogui8 9# Disable PyAutoGUI fail-safe10pyautogui.FAILSAFE = False11 12 13class HandMouseController(VideoProcessorBase):14    def __init__(self):15        # Initialize MediaPipe Hands with specific parameters16        self.hands = mp.solutions.hands.Hands(17            static_image_mode=False, max_num_hands=1, min_detection_confidence=0.7, min_tracking_confidence=0.718        )19        self.drawing_utils = mp.solutions.drawing_utils20 21        # Get the size of the screen22        self.screen_width, self.screen_height = pyautogui.size()23 24        # Variables to keep track of previous hand position25        self.prev_y = None26 27        # Feature toggles28        self.enable_mouse_control = True29        self.enable_scrolling = True30 31    def recv(self, frame):32        img = frame.to_ndarray(format="bgr24")33        frame = cv2.flip(img, 1)34        frame_height, frame_width, _ = frame.shape35        rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)36        result = self.hands.process(rgb_frame)37        hand_landmarks = result.multi_hand_landmarks38 39        if hand_landmarks:40            for hand_landmark in hand_landmarks:41                self.drawing_utils.draw_landmarks(frame, hand_landmark, mp.solutions.hands.HAND_CONNECTIONS)42                landmarks = hand_landmark.landmark43 44                # Extract finger positions45                finger_tips = [8, 12, 16, 20]46                finger_mcp = [5, 9, 13, 17]47                finger_states = []48 49                for tip, mcp in zip(finger_tips, finger_mcp):50                    # Tip above MCP joint means finger is extended51                    if landmarks[tip].y < landmarks[mcp].y:52                        finger_states.append(1)53                    else:54                        finger_states.append(0)55 56                # Check for Victory sign (Index and middle fingers extended)57                if finger_states == [1, 1, 0, 0]:58                    gesture = "Victory"59                # Check for Spider-Man sign (Index and little fingers extended)60                elif finger_states == [1, 0, 0, 1]:61                    gesture = "Spider-Man"62                else:63                    gesture = "None"64 65                # Get index finger coordinates66                index_finger_tip = landmarks[8]67                x = int(index_finger_tip.x * frame_width)68                y = int(index_finger_tip.y * frame_height)69                index_x = self.screen_width / frame_width * x70                index_y = self.screen_height / frame_height * y71 72                # Mouse Control73                if self.enable_mouse_control:74                    # Check for click gesture (thumb and index finger close together)75                    thumb_tip = landmarks[4]76                    thumb_x = int(thumb_tip.x * frame_width)77                    thumb_y = int(thumb_tip.y * frame_height)78                    thumb_index_distance = np.hypot(x - thumb_x, y - thumb_y)79 80                    if thumb_index_distance < 40:81                        # Click action82                        pyautogui.click()83                        pyautogui.sleep(1)84                    elif thumb_index_distance < 100:85                        # print("distance:", thumb_index_distance)86                        # Move cursor87                        pyautogui.moveTo(index_x, index_y)88 89                # Scrolling Control90                if self.enable_scrolling and gesture in ["Victory", "Spider-Man"]:91                    # Get current y position92                    current_y = landmarks[0].y  # Use wrist position93 94                    if self.prev_y is not None:95                        delta_y = self.prev_y - current_y96                        scroll_amount = delta_y * 1000  # Adjust scroll sensitivity97 98                        if abs(scroll_amount) > 5:99                            pyautogui.scroll(int(scroll_amount))100 101                    self.prev_y = current_y102                else:103                    self.prev_y = None104 105        return av.VideoFrame.from_ndarray(frame, format="bgr24")106 107 108def main():109    st.set_page_config(page_title="Virtual Mouse Controller", layout="wide")110    st.title("Virtual Mouse Controller")111 112    st.write(113        """114    Control your computer using hand gestures detected by your webcam.115 116    ### Instructions:117 118    - **Move Cursor** (:pinching_hand:): Hold your index finger up and move your hand to move the cursor.119    - **Click** (:ok_hand:): Bring your thumb and index finger close together to click.120    - **Scroll**:121        - **Victory Sign** (:v:): Extend your index and middle fingers to enable scrolling.122        - **Spider-Man Sign** (:the_horns:): Extend your index and little fingers to enable scrolling.123        - Move your hand **up** or **down** to scroll.124    - **PS**: 125        - Ensure good lighting and keep your hand within the webcam's view.126        - Currently PyAutoGUI doesn't support remote/headless machines. Clone the project and run it in your local machine.127    """128    )129 130    st.sidebar.title("Settings")131    enable_mouse = st.sidebar.checkbox("Enable Mouse Control", value=True)132    enable_scroll = st.sidebar.checkbox("Enable Scrolling", value=True)133 134    # Start the webcam stream with the HandMouseController135    webrtc_ctx = webrtc_streamer(136        key="hand-mouse",137        video_processor_factory=HandMouseController,138        media_stream_constraints={139            "video": {140                "width": {"ideal": 1280},141                "height": {"ideal": 720}142            },143            "audio": False,144        },145        async_processing=True,146        video_html_attrs={147            "style": {"width": "100%", "height": "auto"},148            "controls": False,149            "autoPlay": True,150        },151    )152 153    if webrtc_ctx.video_processor:154        webrtc_ctx.video_processor.enable_mouse_control = enable_mouse155        webrtc_ctx.video_processor.enable_scrolling = enable_scroll156 157    st.sidebar.markdown("---")158    st.sidebar.markdown(159        """160    Developed with ❤️  by **OpenCV University**161 162    **Tools:**163 164    - OpenCV165    - PyAutoGUI166    - MediaPipe167    - Streamlit168    """169    )170 171 172if __name__ == "__main__":173    main()174