""" Head Cursor Control System with Personalized Calibration --------------------------------------------------------- Uses your webcam to track head movement for cursor control and hand gestures for clicking and scrolling. Features: - Personal calibration: maps YOUR comfortable head range to full screen - Adaptive learning: automatically adjusts sensitivity based on usage - Simple robust pinch for clicking - Pinch & hold: HEAD movement controls scrolling (not hand) - Dynamic smoothing based on sensitivity Controls: - Move your head to control the cursor - Quick pinch = click - Pinch & hold: HEAD up/down = scroll (cursor freezes) - Press 'C' to recalibrate - Press '+' / '-' to adjust sensitivity - Press 'L' to toggle adaptive learning - Press ESC to exit """ import cv2 import numpy as np import pyautogui import time import urllib.request import os import json from collections import deque # MediaPipe Tasks API import mediapipe as mp from mediapipe.tasks import python as mp_tasks from mediapipe.tasks.python import vision # --- Configuration --- CAMERA_INDEX = 0 # Base smoothing BASE_SMOOTHING = 0.20 MIN_SMOOTHING = 0.10 MAX_SMOOTHING = 0.35 # Simplified pinch detection PINCH_THRESHOLD = 0.055 # Single threshold for detection PINCH_RELEASE_THRESHOLD = 0.08 # Larger threshold to release (hysteresis) CLICK_COOLDOWN = 0.25 HAND_DETECTION_INTERVAL = 2 # Calibration settings CALIBRATION_HOLD_TIME = 1.5 CALIBRATION_PADDING = 0.03 # Minimum range thresholds MIN_X_RANGE = 0.15 MIN_Y_RANGE = 0.12 # Sensitivity settings DEFAULT_SENSITIVITY = 0.7 MIN_SENSITIVITY = 0.3 MAX_SENSITIVITY = 1.5 SENSITIVITY_STEP = 0.1 # Adaptive learning settings LEARNING_ENABLED = True LEARNING_WINDOW = 300 EDGE_THRESHOLD = 0.05 SENSITIVITY_ADJUST_RATE = 0.02 # Scroll settings - HEAD-based scrolling PINCH_HOLD_TIME = 0.4 # Time to hold pinch before scroll mode SCROLL_SPEED = 50 # Scroll speed multiplier SCROLL_DEAD_ZONE = 0.01 # Minimum head movement to scroll # Model paths MODEL_DIR = os.path.dirname(os.path.abspath(__file__)) FACE_MODEL_PATH = os.path.join(MODEL_DIR, "face_landmarker.task") HAND_MODEL_PATH = os.path.join(MODEL_DIR, "hand_landmarker.task") CALIBRATION_PATH = os.path.join(MODEL_DIR, "calibration.json") FACE_MODEL_URL = "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task" HAND_MODEL_URL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task" global_frame_counter = 0 def get_next_timestamp(): global global_frame_counter global_frame_counter += 1 return global_frame_counter def calculate_dynamic_smoothing(sensitivity): normalized = (sensitivity - MIN_SENSITIVITY) / (MAX_SENSITIVITY - MIN_SENSITIVITY) smoothing = MIN_SMOOTHING + normalized * (MAX_SMOOTHING - MIN_SMOOTHING) return smoothing def download_models(): for url, path, name in [ (FACE_MODEL_URL, FACE_MODEL_PATH, "Face Landmarker"), (HAND_MODEL_URL, HAND_MODEL_PATH, "Hand Landmarker"), ]: if not os.path.exists(path): print(f"Downloading {name} model...") try: urllib.request.urlretrieve(url, path) print(f" [OK] Downloaded {name}") except Exception as e: print(f" [ERROR] Failed to download {name}: {e}") raise def save_calibration(calibration_data): with open(CALIBRATION_PATH, 'w') as f: json.dump(calibration_data, f, indent=2) print("[OK] Calibration saved!") def load_calibration(): if os.path.exists(CALIBRATION_PATH): try: with open(CALIBRATION_PATH, 'r') as f: data = json.load(f) print("[OK] Loaded previous calibration") return data except: pass return None def expand_range_if_needed(calibration): x_range = calibration["x_max"] - calibration["x_min"] y_range = calibration["y_max"] - calibration["y_min"] center_x = (calibration["x_max"] + calibration["x_min"]) / 2 center_y = (calibration["y_max"] + calibration["y_min"]) / 2 if x_range < MIN_X_RANGE: calibration["x_min"] = max(0.0, center_x - MIN_X_RANGE / 2) calibration["x_max"] = min(1.0, center_x + MIN_X_RANGE / 2) if y_range < MIN_Y_RANGE: calibration["y_min"] = max(0.0, center_y - MIN_Y_RANGE / 2) calibration["y_max"] = min(1.0, center_y + MIN_Y_RANGE / 2) return calibration def run_calibration(cap, face_landmarker): print("\n" + "="*60) print(" CALIBRATION MODE") print("="*60) print(" Follow the on-screen instructions.") print(" TIP: Exaggerate movements for better range!") print("="*60 + "\n") steps = [ ("Look at the CENTER of your screen", "center", None), ("Look UP - tilt head back", "max_up", "y_min"), ("Look DOWN - tilt head forward", "max_down", "y_max"), ("Look LEFT - turn head left", "max_left", "x_min"), ("Look RIGHT - turn head right", "max_right", "x_max"), ] calibration = { "x_min": 1.0, "x_max": 0.0, "y_min": 1.0, "y_max": 0.0, "center_x": 0.5, "center_y": 0.5, "learned_sensitivity": DEFAULT_SENSITIVITY, } for step_idx, (instruction, key, direction) in enumerate(steps): step_start = time.time() collected = [] while True: ret, frame = cap.read() if not ret: continue frame = cv2.flip(frame, 1) h, w = frame.shape[:2] rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) ts = get_next_timestamp() result = face_landmarker.detect_for_video(mp_image, ts) nose_x, nose_y = 0.5, 0.5 detected = False if result.face_landmarks: for lm in result.face_landmarks: nose = lm[4] nose_x, nose_y = nose.x, nose.y detected = True cv2.circle(frame, (int(nose_x*w), int(nose_y*h)), 10, (0,255,255), -1) break elapsed = time.time() - step_start if detected and elapsed > 0.3: collected.append((nose_x, nose_y)) progress = min(elapsed / CALIBRATION_HOLD_TIME, 1.0) # UI overlay = frame.copy() cv2.rectangle(overlay, (0,0), (w,120), (30,30,30), -1) frame = cv2.addWeighted(overlay, 0.8, frame, 0.2, 0) cv2.putText(frame, f"Step {step_idx+1}/{len(steps)}", (20,35), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (150,150,150), 2) cv2.putText(frame, instruction, (20,75), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,255,255), 2) cv2.rectangle(frame, (20,95), (w-20,115), (60,60,60), -1) cv2.rectangle(frame, (20,95), (20+int((w-40)*progress),115), (0,200,100), -1) if not detected: cv2.putText(frame, "Face not detected!", (w//2-100, h//2), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0,0,255), 2) cv2.imshow("Head Cursor Control", frame) if cv2.waitKey(1) & 0xFF == 27: return None if progress >= 1.0 and len(collected) > 5: avg_x = np.mean([v[0] for v in collected]) avg_y = np.mean([v[1] for v in collected]) if key == "center": calibration["center_x"] = avg_x calibration["center_y"] = avg_y elif direction == "x_min": calibration["x_min"] = min(calibration["x_min"], avg_x) elif direction == "x_max": calibration["x_max"] = max(calibration["x_max"], avg_x) elif direction == "y_min": calibration["y_min"] = min(calibration["y_min"], avg_y) elif direction == "y_max": calibration["y_max"] = max(calibration["y_max"], avg_y) print(f" [OK] {instruction}") time.sleep(0.3) break calibration["x_min"] -= CALIBRATION_PADDING calibration["x_max"] += CALIBRATION_PADDING calibration["y_min"] -= CALIBRATION_PADDING calibration["y_max"] += CALIBRATION_PADDING calibration = expand_range_if_needed(calibration) print("\n CALIBRATION COMPLETE!") print(f" Range: X[{calibration['x_min']:.2f}-{calibration['x_max']:.2f}] Y[{calibration['y_min']:.2f}-{calibration['y_max']:.2f}]\n") return calibration class SimplePinchDetector: """Simple but robust pinch detection with hysteresis.""" def __init__(self): self.is_pinching = False self.pinch_distance = 1.0 def detect(self, hand_landmarks): if len(hand_landmarks) < 9: self.is_pinching = False return False, 1.0 thumb_tip = hand_landmarks[4] index_tip = hand_landmarks[8] # Calculate distance dist = np.sqrt((thumb_tip.x - index_tip.x)**2 + (thumb_tip.y - index_tip.y)**2) self.pinch_distance = dist # Hysteresis: different thresholds for entering vs exiting if not self.is_pinching: # Need to be close to START pinching if dist < PINCH_THRESHOLD: self.is_pinching = True else: # Need to be far to STOP pinching if dist > PINCH_RELEASE_THRESHOLD: self.is_pinching = False return self.is_pinching, dist class HeadScrollState: """ Scroll state that uses HEAD movement for scrolling. When pinch is held, cursor freezes and head Y controls scroll. """ def __init__(self): self.pinch_start_time = None self.is_scrolling = False self.scroll_anchor_y = None # Head Y when scroll started self.smoothed_head_y = None def update(self, is_pinching, head_y): """ Update scroll state. head_y: normalized Y position of nose (0=top, 1=bottom) Returns: (trigger_click, is_scrolling, scroll_amount) """ trigger_click = False scroll_amount = 0 if is_pinching: if self.pinch_start_time is None: # Just started pinching self.pinch_start_time = time.time() hold_duration = time.time() - self.pinch_start_time if hold_duration >= PINCH_HOLD_TIME: # Enter/continue scroll mode if not self.is_scrolling: # Just entered scroll mode self.is_scrolling = True self.scroll_anchor_y = head_y self.smoothed_head_y = head_y print("[SCROLL MODE - use head to scroll]") # Smooth head movement self.smoothed_head_y = self.smoothed_head_y * 0.7 + head_y * 0.3 # Calculate scroll based on head displacement from anchor displacement = self.smoothed_head_y - self.scroll_anchor_y if abs(displacement) > SCROLL_DEAD_ZONE: # Head down = scroll down (positive), Head up = scroll up (negative) scroll_amount = int(displacement * SCROLL_SPEED) else: # Released pinch if self.pinch_start_time is not None: hold_duration = time.time() - self.pinch_start_time # Quick pinch = click (only if wasn't scrolling) if hold_duration < PINCH_HOLD_TIME and not self.is_scrolling: trigger_click = True # Reset self.pinch_start_time = None self.is_scrolling = False self.scroll_anchor_y = None return trigger_click, self.is_scrolling, scroll_amount def get_hold_progress(self): if self.pinch_start_time is None or self.is_scrolling: return 0 return min((time.time() - self.pinch_start_time) / PINCH_HOLD_TIME, 1.0) class AdaptiveLearning: def __init__(self, initial_sensitivity): self.sensitivity = initial_sensitivity self.position_history = deque(maxlen=LEARNING_WINDOW) self.edge_hits = 0 self.frames_analyzed = 0 self.enabled = LEARNING_ENABLED def record(self, mapped_x, mapped_y): if not self.enabled: return self.position_history.append((mapped_x, mapped_y)) if mapped_x < EDGE_THRESHOLD or mapped_x > (1 - EDGE_THRESHOLD): self.edge_hits += 1 if mapped_y < EDGE_THRESHOLD or mapped_y > (1 - EDGE_THRESHOLD): self.edge_hits += 1 self.frames_analyzed += 1 def learn(self): if not self.enabled or len(self.position_history) < LEARNING_WINDOW: return self.sensitivity edge_rate = self.edge_hits / max(self.frames_analyzed * 2, 1) if edge_rate > 0.15: self.sensitivity = max(MIN_SENSITIVITY, self.sensitivity - SENSITIVITY_ADJUST_RATE) elif edge_rate < 0.02: self.sensitivity = min(MAX_SENSITIVITY, self.sensitivity + SENSITIVITY_ADJUST_RATE * 0.5) self.edge_hits = 0 self.frames_analyzed = 0 return self.sensitivity def draw_ui(frame, face_result, hand_result, pinch_detector, scroll_state, is_scrolling): display = frame.copy() h, w = display.shape[:2] # Face landmark if face_result and face_result.face_landmarks: for lm in face_result.face_landmarks: if len(lm) > 4: nose = lm[4] cx, cy = int(nose.x * w), int(nose.y * h) # Different color when scrolling color = (255, 165, 0) if is_scrolling else (0, 255, 255) cv2.circle(display, (cx, cy), 10, color, -1) cv2.circle(display, (cx, cy), 14, (0, 200, 200), 2) # Draw scroll indicator when scrolling if is_scrolling and scroll_state.scroll_anchor_y: anchor_y = int(scroll_state.scroll_anchor_y * h) cv2.line(display, (cx - 30, anchor_y), (cx + 30, anchor_y), (255, 165, 0), 2) cv2.arrowedLine(display, (cx, anchor_y), (cx, cy), (255, 165, 0), 2, tipLength=0.3) # Hand landmarks if hand_result and hand_result.hand_landmarks: for hand_lm in hand_result.hand_landmarks: for i, lm in enumerate(hand_lm): px, py = int(lm.x * w), int(lm.y * h) color = (255, 0, 255) if i in [4, 8] else (180, 180, 180) cv2.circle(display, (px, py), 3, color, -1) if len(hand_lm) > 8: thumb = hand_lm[4] index = hand_lm[8] t_x, t_y = int(thumb.x * w), int(thumb.y * h) i_x, i_y = int(index.x * w), int(index.y * h) if is_scrolling: line_color = (255, 165, 0) elif pinch_detector.is_pinching: line_color = (0, 0, 255) else: line_color = (0, 255, 0) cv2.line(display, (t_x, t_y), (i_x, i_y), line_color, 3) # Hold progress circle progress = scroll_state.get_hold_progress() if progress > 0 and not is_scrolling: mid_x, mid_y = (t_x + i_x) // 2, (t_y + i_y) // 2 radius = int(30 * progress) cv2.circle(display, (mid_x, mid_y), radius, (0, 165, 255), 3) return display def main(): pyautogui.FAILSAFE = False pyautogui.PAUSE = 0 screen_w, screen_h = pyautogui.size() print(f"Screen: {screen_w}x{screen_h}") download_models() face_options = vision.FaceLandmarkerOptions( base_options=mp_tasks.BaseOptions(model_asset_path=FACE_MODEL_PATH), running_mode=vision.RunningMode.VIDEO, num_faces=1, min_face_detection_confidence=0.5, min_face_presence_confidence=0.5, min_tracking_confidence=0.5, ) face_landmarker = vision.FaceLandmarker.create_from_options(face_options) hand_options = vision.HandLandmarkerOptions( base_options=mp_tasks.BaseOptions(model_asset_path=HAND_MODEL_PATH), running_mode=vision.RunningMode.VIDEO, num_hands=1, min_hand_detection_confidence=0.6, min_hand_presence_confidence=0.6, min_tracking_confidence=0.6, ) hand_landmarker = vision.HandLandmarker.create_from_options(hand_options) cap = cv2.VideoCapture(CAMERA_INDEX) if not cap.isOpened(): print("Error: Could not open camera.") return cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) cap.set(cv2.CAP_PROP_FPS, 30) calibration = load_calibration() if calibration is None: calibration = run_calibration(cap, face_landmarker) if calibration is None: cap.release() cv2.destroyAllWindows() return save_calibration(calibration) else: calibration = expand_range_if_needed(calibration) x_min, x_max = calibration["x_min"], calibration["x_max"] y_min, y_max = calibration["y_min"], calibration["y_max"] x_range, y_range = x_max - x_min, y_max - y_min sensitivity = calibration.get("learned_sensitivity", DEFAULT_SENSITIVITY) prev_x, prev_y = screen_w / 2, screen_h / 2 last_click_time = 0 frame_count = 0 last_hand_result = None pinch_detector = SimplePinchDetector() scroll_state = HeadScrollState() learner = AdaptiveLearning(sensitivity) last_learn_time = time.time() print("\n" + "="*55) print(" HEAD CURSOR CONTROL") print("="*55) print(" - Move head = move cursor") print(" - Quick pinch = click") print(" - Hold pinch + move HEAD = scroll") print(" - C=recal +/-=sens L=learn ESC=exit") print("="*55 + "\n") while cap.isOpened(): ret, frame = cap.read() if not ret: break frame = cv2.flip(frame, 1) rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) ts = get_next_timestamp() h, w = frame.shape[:2] smoothing = calculate_dynamic_smoothing(sensitivity) # Face tracking face_result = face_landmarker.detect_for_video(mp_image, ts) head_y = 0.5 # For scroll mapped_x, mapped_y = 0.5, 0.5 if face_result.face_landmarks: for lm in face_result.face_landmarks: nose = lm[4] raw_x, raw_y = nose.x, nose.y head_y = raw_y # Raw head Y for scrolling # Calculate mapped position eff_x_range = x_range / sensitivity eff_y_range = y_range / sensitivity center_x = (x_min + x_max) / 2 center_y = (y_min + y_max) / 2 eff_x_min = center_x - eff_x_range / 2 eff_x_max = center_x + eff_x_range / 2 eff_y_min = center_y - eff_y_range / 2 eff_y_max = center_y + eff_y_range / 2 clamped_x = np.clip(raw_x, eff_x_min, eff_x_max) clamped_y = np.clip(raw_y, eff_y_min, eff_y_max) mapped_x = (clamped_x - eff_x_min) / eff_x_range if eff_x_range > 0.01 else 0.5 mapped_y = (clamped_y - eff_y_min) / eff_y_range if eff_y_range > 0.01 else 0.5 learner.record(mapped_x, mapped_y) break # Hand tracking if frame_count % HAND_DETECTION_INTERVAL == 0: hand_result = hand_landmarker.detect_for_video(mp_image, get_next_timestamp()) last_hand_result = hand_result else: hand_result = last_hand_result is_pinching = False if hand_result and hand_result.hand_landmarks: for hand_lm in hand_result.hand_landmarks: is_pinching, _ = pinch_detector.detect(hand_lm) break else: pinch_detector.is_pinching = False # Update scroll state with HEAD position trigger_click, is_scrolling, scroll_amount = scroll_state.update(is_pinching, head_y) # Handle click if trigger_click: now = time.time() if now - last_click_time > CLICK_COOLDOWN: pyautogui.click() last_click_time = now print("[CLICK]") # Handle cursor movement (freeze when scrolling) if not is_scrolling: target_x = mapped_x * screen_w target_y = mapped_y * screen_h curr_x = prev_x + (target_x - prev_x) * smoothing curr_y = prev_y + (target_y - prev_y) * smoothing pyautogui.moveTo(int(curr_x), int(curr_y)) prev_x, prev_y = curr_x, curr_y # Handle scroll if scroll_amount != 0: pyautogui.scroll(-scroll_amount) # Learning now = time.time() if now - last_learn_time > 10: new_sens = learner.learn() if new_sens != sensitivity: sensitivity = new_sens calibration["learned_sensitivity"] = sensitivity save_calibration(calibration) last_learn_time = now # UI display = draw_ui(frame, face_result, hand_result, pinch_detector, scroll_state, is_scrolling) # Status if is_scrolling: direction = "v" if scroll_amount > 0 else ("^" if scroll_amount < 0 else "-") status, color = f"SCROLL {direction}", (255, 165, 0) elif scroll_state.get_hold_progress() > 0: status, color = f"HOLD {scroll_state.get_hold_progress():.0%}", (0, 165, 255) elif is_pinching: status, color = "PINCH", (0, 0, 255)z else: status, color = "Ready", (0, 255, 0) cv2.putText(display, status, (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, color, 2) # Info info = f"Sens:{sensitivity:.1f} Smooth:{smoothing:.2f}" cv2.putText(display, info, (20, h-20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (180,180,180), 1) cv2.putText(display, "C=recal +/-=sens L=learn", (w-230, h-20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (150,150,150), 1) cv2.imshow("Head Cursor Control", display) key = cv2.waitKey(1) & 0xFF if key == 27: break elif key == ord('c') or key == ord('C'): new_cal = run_calibration(cap, face_landmarker) if new_cal: calibration = new_cal calibration["learned_sensitivity"] = sensitivity save_calibration(calibration) x_min, x_max = calibration["x_min"], calibration["x_max"] y_min, y_max = calibration["y_min"], calibration["y_max"] x_range, y_range = x_max - x_min, y_max - y_min elif key == ord('+') or key == ord('='): sensitivity = min(MAX_SENSITIVITY, sensitivity + SENSITIVITY_STEP) learner.sensitivity = sensitivity elif key == ord('-') or key == ord('_'): sensitivity = max(MIN_SENSITIVITY, sensitivity - SENSITIVITY_STEP) learner.sensitivity = sensitivity elif key == ord('l') or key == ord('L'): learner.enabled = not learner.enabled print(f"Learning: {'ON' if learner.enabled else 'OFF'}") frame_count += 1 calibration["learned_sensitivity"] = sensitivity save_calibration(calibration) cap.release() cv2.destroyAllWindows() face_landmarker.close() hand_landmarker.close() print("\nStopped.") if __name__ == "__main__": main()