mediapipe 实现AI 虚拟鼠标

主要使用了mediapipe、opencv和autopy实现，通过抓取关键信息建立一些函数来实现虚拟鼠标。代码我自己分为两个部分，第一个部分是写的一个封装好的类，第二个直接调用即可。

第一个文件名为handdetect.py 第二个是AI mouse.py 提到这个主要是因为我在第二个代码里面把第一部分的代码全部封装了（如果是小白的话把文件名改一下放在同一个目录下即可）

opencv应该都可以安装，可能大家安装autopy时候会出问题，首先它是需要python版本不能大于3.8

，所以python版本小于等于3.8即可正常安装使用pip install autopy，不行的话只能去手动安装了。

评论区欢迎讨论！

handdetect.py

import cv2
import mediapipe as mp
import time
import math


class HandDetector():
    def __init__(self,static_img_mode=False,max_num_hands=2,model_complexity=1,min_detection_confidence=0.5,
                 min_traching_confidence=0.5):
        self.mdoe = static_img_mode
        self.maxHand = max_num_hands
        self.modecomplex = model_complexity
        self.min_det_con = min_detection_confidence
        self.min_track_con = min_traching_confidence

        self.mphands = mp.solutions.hands
        self.handdetect = self.mphands.Hands()
        self.draw = mp.solutions.drawing_utils
        self.ID =[4,8,12,16,20]
    #列出全部手指的节点
    def findhands(self,img):
        RGB_img = cv2.cvtColor (img, cv2.COLOR_BGR2RGB)

        # 检测手部打印信息
        self.resturns = self.handdetect.process (RGB_img)

        x_list = []
        y_list = []
        self.lmist = []

        if (self.resturns.multi_hand_landmarks):
            for landmarks in self.resturns.multi_hand_landmarks:
                self.draw.draw_landmarks (img, landmarks, mp.solutions.hands.HAND_CONNECTIONS,
                                     mp.solutions.drawing_styles.get_default_pose_landmarks_style ( ))
                for id, landmark in enumerate (landmarks.landmark):
                    # print(id,landmark)

                    H, W, _ = img.shape
                    x = int (landmark.x * W)
                    y = int (landmark.y * H)
                    x_list.append (x)
                    y_list.append (y)
                    #print (id, x, y)

                    self.lmist.append([id,x,y])
                x_min, y_min = min (x_list), min (y_list)
                x_max, y_max = max (x_list), max (y_list)
                cv2.rectangle (img, (x_min - 20, y_min - 20), (x_max + 20, y_max + 20), (0, 220, 220), 2)
        return img,self.lmist

    #列出指定手指
    def findfingers(self,img,fingerID):
        x , y = self.lmist[fingerID][1:]
        cv2.circle(img,(x,y),10,(255,1,1),5,cv2.FILLED)
        return img

    #手指之间的距离
    def finddistance(self,img,p1,p2):
        x1 , y1 = self.lmist[p1][1:]
        x2 , y2 = self.lmist[p2][1:]

        distance = math.hypot(x2-x1,y2-y1)

        cv2.line(img,(x1,y1),(x2,y2),(25,100,100),3)#划线
        return img,distance

    #判断手指竖立
    def fingersup(self):
        finger_id =[]
        for id in range(0,5):
            if (self.lmist[self.ID[id]][2] < self.lmist[self.ID[id]-2][2]):
                finger_id.append(1)
            else:
                finger_id.append(0)
        return finger_id

def main():
    cap = cv2.VideoCapture (0)
    pTime = 0
    handdetect = HandDetector()

    while True:
        success, img = cap.read ( )

        #调用封装函数
        img ,position_list = handdetect.findhands(img)
        if (len(position_list)!=0):
            #print(position_list)

            img = handdetect.findfingers(img,8)
            img = handdetect.findfingers(img,12)

            img ,distance = handdetect.finddistance(img,8,12)
            # print(distance)
            finger =handdetect.fingersup()
            print(finger)




        #设置fps
        cTime = time.time ( )
        fps = 1 / (cTime - pTime)  # 计算fps
        pTime = cTime  # 更新下一帧图像处理的起始时间
        # 把fps值显示在图像上,img画板;fps变成字符串;显示的位置;设置字体;字体大小;字体颜色;线条粗细
        cv2.putText (img, f'FPS: {str (int (fps))}', (10, 50), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 0), 3)

        cv2.imshow ("image", img)
        cv2.waitKey (1)


    cap.release ( )
    cv2.destoryALLwindows ( )


if __name__ == "__main__":
    main()

第二部分

AI mouse.py

import handdetect as htm
import cv2
import time
import autopy
import numpy as np

Wacreen,Hacreen =autopy.screen.size()
wCam,hCam =1023,880

cap = cv2.VideoCapture (0)
pTime = 0
handdetect = htm.HandDetector()

while True:
    success, img = cap.read ( )

    # 调用封装函数
    #找到手部信息
    img, position_list = handdetect.findhands (img)
    #找到食指和中指
    if (len (position_list) != 0):
        x1 , y1 = position_list[8][1:]
        x2 , y2 = position_list[12][1:]
        # 先画一个执行区域
        cv2.rectangle (img, (20, 20), (wCam - 20, hCam - 20), (20, 240, 89), 2)
        #检测哪些手指是立着的
        finger = handdetect.fingersup()
        #只有食指时移动鼠标
        if finger[1] == 1 and finger[2] == 0 and finger[3] == 0 and finger[4] == 0:
            x3 = np.interp(x1,(20,wCam-20),(0,Wacreen))
            y3 = np.interp(y1,(20,hCam-20),(0,Hacreen))
            cv2.circle(img,(x1,y1),10,(90,89,90),cv2.FILLED)
        # 鼠标移动
            autopy.mouse.move(Wacreen-x3,y3) #因为镜头反转的原理所以需要Wacreen-x3
        #食指和中指合并的时候点击
        if finger[1] == 1 and finger[2] == 1 and finger[3] == 0 and finger[4] == 0:
            img,distance = handdetect.finddistance(img,8,12)
            if distance < 80:
                x_center = (x1 + x2) // 2
                y_center = (y1 + y2) // 2
                cv2.circle(img,(x_center,y_center),10,(1,1,1),cv2.FILLED)
                autopy.mouse.click(None)
        #食指和小拇指竖立是右键
        if finger[1] == 1 and finger[2] == 0 and finger[3] == 0 and finger[4] == 1:
            autopy.mouse.click(autopy.mouse.Button.RIGHT)


    #设置fps
    cTime = time.time ( )
    fps = 1 / (cTime - pTime)  # 计算fps
    pTime = cTime  # 更新下一帧图像处理的起始时间
    # 把fps值显示在图像上,img画板;fps变成字符串;显示的位置;设置字体;字体大小;字体颜色;线条粗细
    cv2.putText (img, f'FPS: {str (int (fps))}', (10, 50), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 0), 3)

    img = cv2.resize(img,(1024,900))
    cv2.imshow ("image", img)
    if ord('q') == cv2.waitKey(1):
        break
cap.release ( )