Operación de simulación del mouse Python

Operación de simulación del mouse Python

Simplemente registre las coordenadas del mouse y calcule el desplazamiento de la imagen debajo del mouse para completar una calibración de posición simple, que se utiliza para simular la operación del mouse en la interfaz de microdesplazamiento.
Enlace de GitHub
https://github.com/WindSnowLi/My-python-tools/tree/main/PseudoOperation

Parte de registro

# -*- coding:utf-8 -*-
import datetime
import os
from threading import Thread

import pynput
from PIL import ImageGrab
from pynput.keyboard import Key

# 监听鼠标
last_time = datetime.datetime.now()
operate_record = []
id = 0
status_flag = True


# 截取指定位置
def make_screenshot(x1, y1, x2, y2):
    """截图

    :param x1: 开始截图的x1坐标
    :param y1: 开始截图的x1标
    :param x2: 开始截图的x2坐标
    :param y2: 结束截图的y2坐标
    :return: None
    """
    # id: 图片ID
    global id
    bbox = (x1, y1, x2, y2)
    im = ImageGrab.grab(bbox)
    im.save('./pic/' + str(id) + '.png')  # 保存截图文件的路径


# 监听鼠标移动，并记录移动信息
def on_move(x, y):
    global status_flag
    if not status_flag:
        return status_flag
    global last_time
    global id
    global operate_record
    # 防止读取的鼠标位置越界
    if y > 1080:
        y = 1080
    if x > 1920:
        x = 1920
    if x < 0:
        x = 0
    if y < 0:
        y = 0
    print('Pointer moved to {0}'.format((x, y)))
    id += 1
    # 边缘位置不再进行截图
    if id % 100 == 0 and (x >= 150 or x <= 1920 - 150) and (y > 100 or y < 1080 - 100):
        make_screenshot(x - 75, y - 50, x + 75, y + 50)
    single = {
    
    'id': id, 'x': x, 'y': y, "event": "move", "button": "", 'action': '',
              'time': (datetime.datetime.now() - last_time).total_seconds()}
    last_time = datetime.datetime.now()
    operate_record.append(single)


# 监听点击信息
def on_click(x, y, button, pressed):
    global status_flag
    if not status_flag:
        return status_flag
    global last_time
    global id
    global operate_record
    # 监听鼠标点击
    print('{0} at {1}'.format('Pressed' if pressed else 'Released', (x, y)))
    id += 1
    single = {
    
    'id': id, 'x': x, 'y': y, "event": "click", "button": str(button),
              'action': 'pressed' if pressed else 'released',
              'time': (datetime.datetime.now() - last_time).total_seconds()}
    last_time = datetime.datetime.now()
    operate_record.append(single)
    make_screenshot(x - 75, y - 50, x + 75, y + 50)


# 监听键盘按键按下
def on_press(key):
    global status_flag
    if not status_flag:
        return status_flag
    # 监听按键
    print('{0} pressed'.format(key))


# 监听按键释放信息
def on_release(key):
    global status_flag
    global operate_record
    # 监听释放
    print('{0} release'.format(key))
    # 若按下的为ESC键，终止程序
    if key == Key.esc:
        with open('./record.txt', 'w') as fp:
            for i in operate_record:
                fp.write(str(i) + '\n')
        print("end")
        status_flag = False
        return status_flag


def listener_mouse():
    with pynput.mouse.Listener(on_move=on_move, on_click=on_click) as listener:
        listener.join()


def listener_key():
    with pynput.keyboard.Listener(on_press=on_press, on_release=on_release)as listener:
        listener.join()


# 一个鼠标监听器是一个线程。线程，所有的回调将从线程调用。从任何地方调用pynput.mouse.Listener.stop，或者调用pynput.mouse.Listener.StopException或从回调中返回False来停止监听器。


if __name__ == '__main__':
    if not os.path.exists('./pic'):
        os.mkdir('./pic')
    t1 = Thread(target=listener_mouse)
    t2 = Thread(target=listener_key)
    t1.start()
    t2.start()

Parte de la operación

# -*- coding:utf-8 -*-


# 控制鼠标
# 读鼠标坐标
import json
import os
import time
from threading import Thread

import cv2
from PIL import ImageGrab
from pynput.keyboard import Key

import pynput

status_flag = True
mouse = pynput.mouse.Controller()
calibration = [0, 0]


def make_screenshot(x1, y1, x2, y2):
    """截图

    :param x1: 开始截图的x1坐标
    :param y1: 开始截图的x1标
    :param x2: 开始截图的x2坐标
    :param y2: 结束截图的y2坐标
    :return: None
    """
    # id: 图片ID
    global id
    bbox = (x1, y1, x2, y2)
    im = ImageGrab.grab(bbox)
    im.save('./temp/present.png')  # 保存截图文件的路径


def identify_pictures(img1Path, img2Path):
    img = cv2.imread(img1Path, 0)
    img2 = img.copy()
    template = cv2.imread(img2Path, 0)
    w, h = template.shape[::-1]

    meth = 'cv2.TM_CCOEFF'
    img = img2.copy()

    method = eval(meth)

    res = cv2.matchTemplate(img, template, method)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
    return max_loc


def setPosition(x, y):
    global mouse
    # 本人电脑点击时鼠标坐标会放大偏移，大概会放大1.25倍
    mouse.position = (x / 1.25, y / 1.25)


def setLeftPress():
    global mouse
    mouse.press(pynput.mouse.Button.left)


def setLeftRelease():
    global mouse
    mouse.release(pynput.mouse.Button.left)


def setRightPress():
    global mouse
    mouse.press(pynput.mouse.Button.right)


def setRightRelease():
    global mouse
    mouse.release(pynput.mouse.Button.right)


def on_press(key):
    # 监听按键
    print('{0} pressed'.format(key))


def on_release(key):
    global status_flag
    # 监听释放
    print('{0} release'.format(key))
    if key == Key.esc:
        status_flag = False
        return status_flag


# 通过查找原始鼠标下的图片位置与现在鼠标下的位置进行对比，计算偏移量
def calibration_offset(i):
    make_screenshot(i['x'] - 150 + calibration[0], i['y'] - 100 + calibration[1], i['x'] + 150 + calibration[0],
                    i['y'] + 100 + calibration[1])
    inner_top_left = identify_pictures('./temp/present.png', './pic/' + str(i['id']) + '.png')
    outside = (
        i['x'] - 150 + inner_top_left[0] + calibration[0], i['y'] - 100 + inner_top_left[1] + calibration[1])

    calibration[0] = outside[0] - (i['x'] - 75)
    calibration[1] = outside[1] - (i['y'] - 50)


def operate_mouse():
    global status_flag
    record = []
    with open("./record.txt", 'r') as fp:
        for line in fp:
            record.append(json.loads(line.replace("\'", "\"")))
    for i in record:
        # print(i)
        if not status_flag:
            break
        if i['event'] == 'move':
            if i['id'] % 100 == 0 and (i['x'] >= 200 or i['x'] <= 1920 - 200) and (i['y'] > 100 or i['y'] < 1080 - 100):
                calibration_offset(i)
            i['x'] = i['x'] + calibration[0]
            i['y'] = i['y'] + calibration[1]
            setPosition(i['x'], i['y'])
        elif i['event'] == 'click':
            calibration_offset(i)
            setPosition(i['x'] + calibration[0], i['y'] + calibration[1])
            if i['action'] == 'pressed':
                if i['button'] == 'Button.left':
                    setLeftPress()
                elif i['button'] == 'Button.right':
                    setRightPress()
            elif i['action'] == 'released':
                if i['button'] == 'Button.left':
                    setLeftRelease()
                elif i['button'] == 'Button.right':
                    setRightRelease()
        time.sleep(i['time'])
    status_flag = False
    print("end")


def listener_key():
    with pynput.keyboard.Listener(on_press=on_press, on_release=on_release)as listener:
        listener.join()


if __name__ == '__main__':
    if not os.path.exists('./temp'):
        os.mkdir('./temp')
    t1 = Thread(target=operate_mouse)
    t2 = Thread(target=listener_key)
    t1.start()
    t2.start()