From 947c709c74d837f497eaaf5fe45e8ea2aa99a457 Mon Sep 17 00:00:00 2001 From: shibatachun Date: Tue, 2 Apr 2024 11:58:37 -0400 Subject: [PATCH 1/4] added code for simulating mouse to implement add to playlist feature --- officialVersion/simulateMouse.py | 137 +++++++++++++++++++++++++++++++ officialVersion/visual.py | 1 + 2 files changed, 138 insertions(+) create mode 100644 officialVersion/simulateMouse.py diff --git a/officialVersion/simulateMouse.py b/officialVersion/simulateMouse.py new file mode 100644 index 000000000..98def4f68 --- /dev/null +++ b/officialVersion/simulateMouse.py @@ -0,0 +1,137 @@ +import cv2 +import numpy as np +import mediapipe as mp +from math import sqrt +import urllib.request +from pynput.mouse import Button, Controller + + +cap = cv2.VideoCapture(0, cv2.CAP_DSHOW) + +cap.set(3, 480) +cap.set(4, 360) + +screen_width = 1920 +screen_height = 1080 + +mpHands = mp.solutions.hands +hands = mpHands.Hands(max_num_hands=1) +mpDraw = mp.solutions.drawing_utils + +dist = 0 + +mouse = Controller() + +mouseOldLoc = np.array([0, 0]) +mouseLoc = np.array([0, 0]) +dampingFactor = 2 + +db_click_flag = 0 +scroll_down_flag = 0 +click_flag = 0 +scroll_up_flag = 0 + +while True: + mouse.release(Button.left) + ret, frame = cap.read() + + + + + + frameRGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + cv2.rectangle(frame, (10, 10), (20, 20), (0, 255, 255), -1) + cv2.putText(frame, "Double Click", (30, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 255, 255), 1) + + cv2.rectangle(frame, (200, 10), (210, 20), (0, 255, 0), -1) + cv2.putText(frame, "Click", (220, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 255, 0), 1) + + cv2.rectangle(frame, (300, 10), (310, 20), (0, 0, 255), -1) + cv2.putText(frame, "Scroll Down", (320, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 1) + + cv2.rectangle(frame, (490, 10), (500, 20), (255, 255, 255), 1) + cv2.putText(frame, "Cursor", (510, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (255, 255, 255), 1) + + results = hands.process(frameRGB) + + if results.multi_hand_landmarks: + for handLms in results.multi_hand_landmarks: + for id, lm in enumerate(handLms.landmark): + h, w, c = frame.shape + + try: + if id == 4: + thumb_cx, thumb_cy = int(lm.x * w), int(lm.y * h) + if id == 8: + index_cx, index_cy = int(lm.x * w), int(lm.y * h) + if id == 20: + pinky_cx, pinky_cy = int(lm.x * w), int(lm.y * h) + if id == 12: + mid_tip_cx, mid_tip_cy = int(lm.x * w), int(lm.y * h) + if id == 16: + ring_tip_cx, ring_tip_cy = int(lm.x * w), int(lm.y * h) + + cv2.circle(frame, (index_cx, index_cy), 5, (0, 255, 0), -1) + cv2.circle(frame, (mid_tip_cx, mid_tip_cy), 5, (0, 255, 255), -1) + cv2.circle(frame, (pinky_cx, pinky_cy), 5, (0, 0, 255), -1) + cv2.circle(frame, (ring_tip_cx, ring_tip_cy), 5, (125, 0, 255), -1) + + center_thumb_index_x = int((thumb_cx + index_cx) / 2) + center_thumb_index_y = int((thumb_cy + index_cy) / 2) + + cv2.circle(frame, (center_thumb_index_x, center_thumb_index_y), 4, (255, 255, 255), 1) + + dist_thumb_index = sqrt((thumb_cx - index_cx) ** 2 + (thumb_cy - index_cy) ** 2) + dist_thumb_pinky = sqrt((thumb_cx - pinky_cx) ** 2 + (thumb_cy - pinky_cy) ** 2) + dist_thumb_mid = sqrt((thumb_cx - mid_tip_cx) ** 2 + (thumb_cy - mid_tip_cy) ** 2) + dist_thumb_ring = sqrt((thumb_cx - ring_tip_cx) ** 2 + (thumb_cy - ring_tip_cy) ** 2) + + mouseLoc = mouseOldLoc + ( + (center_thumb_index_x, center_thumb_index_y) - mouseOldLoc) / dampingFactor + + mouse_x = int((mouseLoc[0] * screen_width / 600)) + mouse_y = int((mouseLoc[1] * screen_height / 360)) + + mouse.position = (mouse_x, mouse_y) + mouseOldLoc = mouseLoc + + if dist_thumb_index > 0 and dist_thumb_index < 15: + db_click_flag = 0 + scroll_down_flag = 0 + scroll_up_flag = 0 + cv2.putText(frame, "Click", (220, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 255, 0), 2) + mouse.click(Button.left, 1) + + + + + elif dist_thumb_mid > 0 and dist_thumb_mid < 15: + if db_click_flag == 0: + db_click_flag = 1 + + cv2.putText(frame, "Double Click", (30, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, + (0, 255, 255), 2) + mouse.click(Button.left, 2) + elif dist_thumb_pinky > 0 and dist_thumb_pinky < 15: + if scroll_down_flag == 0: + db_click_flag = 0 + scroll_down_flag = 1 + cv2.putText(frame, "Scroll Down", (320, 22), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), + 2) + mouse.scroll(0, -2) + elif dist_thumb_ring > 0 and dist_thumb_ring < 15: + if scroll_up_flag == 0: + db_click_flag = 0 + scroll_up_flag = 1 + mouse.scroll(0, 2) + + + except: + pass + + cv2.imshow("frame", frame) + if cv2.waitKey(1) == ord('q'): + break + +cv2.destroyAllWindows() \ No newline at end of file diff --git a/officialVersion/visual.py b/officialVersion/visual.py index 8ea78f8a3..17499e580 100644 --- a/officialVersion/visual.py +++ b/officialVersion/visual.py @@ -17,6 +17,7 @@ def print_result(result: GestureRecognizerResult, output_image: mp.Image, timestamp_ms: int): print('gesture recognition result: {}'.format(result)) + model_path = 'gesture_recognizer.task' base_options = BaseOptions(model_asset_path=model_path) options = GestureRecognizerOptions( From f8626df28de9ecf46add9daa1f505899d14911c3 Mon Sep 17 00:00:00 2001 From: shibatachun Date: Tue, 2 Apr 2024 11:59:52 -0400 Subject: [PATCH 2/4] added code for simulating mouse to implement add to playlist feature --- officialVersion/simulateMouse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/officialVersion/simulateMouse.py b/officialVersion/simulateMouse.py index 98def4f68..361451605 100644 --- a/officialVersion/simulateMouse.py +++ b/officialVersion/simulateMouse.py @@ -1,3 +1,5 @@ +##These code is modified from https://github.com/Ashvin-G/Virtual-Mouse + import cv2 import numpy as np import mediapipe as mp From 25c8e55925680d57d595137353f4c0efb1b18742 Mon Sep 17 00:00:00 2001 From: shibatachun Date: Tue, 2 Apr 2024 12:08:07 -0400 Subject: [PATCH 3/4] Fix the resolution error, now the cursor will cover all the screen --- officialVersion/simulateMouse.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/officialVersion/simulateMouse.py b/officialVersion/simulateMouse.py index 361451605..b376bdcf3 100644 --- a/officialVersion/simulateMouse.py +++ b/officialVersion/simulateMouse.py @@ -6,15 +6,15 @@ from math import sqrt import urllib.request from pynput.mouse import Button, Controller - +from screeninfo import get_monitors cap = cv2.VideoCapture(0, cv2.CAP_DSHOW) cap.set(3, 480) cap.set(4, 360) -screen_width = 1920 -screen_height = 1080 +#Set your resolution here +screen_width, screen_height = get_monitors()[0].width, get_monitors()[0].height mpHands = mp.solutions.hands hands = mpHands.Hands(max_num_hands=1) @@ -91,7 +91,10 @@ mouseLoc = mouseOldLoc + ( (center_thumb_index_x, center_thumb_index_y) - mouseOldLoc) / dampingFactor - + if mouseLoc.any(): + scaled_x = np.interp(mouseLoc[0], [0, w], [0, screen_width]) + scaled_y = np.interp(mouseLoc[1], [0, h], [0, screen_height]) + mouse.position = (scaled_x, scaled_y) mouse_x = int((mouseLoc[0] * screen_width / 600)) mouse_y = int((mouseLoc[1] * screen_height / 360)) From 85df7aa163fdeb8f6e4e2a9ea7dd4086bc3d948f Mon Sep 17 00:00:00 2001 From: shibatachun Date: Tue, 2 Apr 2024 12:17:30 -0400 Subject: [PATCH 4/4] For volume control gesture, set the hotkey control spotify volume instead of control system volume --- gesture/volume.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gesture/volume.py b/gesture/volume.py index 83de98943..7837fba20 100644 --- a/gesture/volume.py +++ b/gesture/volume.py @@ -44,11 +44,11 @@ def main(): # Detect finger pointing up if fingers[1] and all(not f for f in fingers[2:]): - pyautogui.press('volumeup') + pyautogui.hotkey('ctrl', 'down') print("Volume Up") # Detect finger pointing down elif not fingers[1] and all(not f for f in fingers[2:]): - pyautogui.press('volumedown') + pyautogui.hotkey('ctrl', 'up') print("Volume Down") # Print the corresponding number based on the number of fingers stretched out