Python mouse simulation operation

Python mouse simulation operation

Simply record the mouse coordinates and calculate the offset of the picture under the mouse to complete a simple position calibration, which is used to simulate the mouse operation on the micro-offset interface.
GitHub link
https://github.com/WindSnowLi/My-python-tools/tree/main/PseudoOperation

Record part

# -*- coding:utf-8 -*-
import datetime
import os
from threading import Thread

import pynput
from PIL import ImageGrab
from pynput.keyboard import Key

# 监听鼠标
last_time = datetime.datetime.now()
operate_record = []
id = 0
status_flag = True


# 截取指定位置
def make_screenshot(x1, y1, x2, y2):
    """截图

    :param x1: 开始截图的x1坐标
    :param y1: 开始截图的x1标
    :param x2: 开始截图的x2坐标
    :param y2: 结束截图的y2坐标
    :return: None
    """
    # id: 图片ID
    global id
    bbox = (x1, y1, x2, y2)
    im = ImageGrab.grab(bbox)
    im.save('./pic/' + str(id) + '.png')  # 保存截图文件的路径


# 监听鼠标移动，并记录移动信息
def on_move(x, y):
    global status_flag
    if not status_flag:
        return status_flag
    global last_time
    global id
    global operate_record
    # 防止读取的鼠标位置越界
    if y > 1080:
        y = 1080
    if x > 1920:
        x = 1920
    if x < 0:
        x = 0
    if y < 0:
        y = 0
    print('Pointer moved to {0}'.format((x, y)))
    id += 1
    # 边缘位置不再进行截图
    if id % 100 == 0 and (x >= 150 or x <= 1920 - 150) and (y > 100 or y < 1080 - 100):
        make_screenshot(x - 75, y - 50, x + 75, y + 50)
    single = {
    
    'id': id, 'x': x, 'y': y, "event": "move", "button": "", 'action': '',
              'time': (datetime.datetime.now() - last_time).total_seconds()}
    last_time = datetime.datetime.now()
    operate_record.append(single)


# 监听点击信息
def on_click(x, y, button, pressed):
    global status_flag
    if not status_flag:
        return status_flag
    global last_time
    global id
    global operate_record
    # 监听鼠标点击
    print('{0} at {1}'.format('Pressed' if pressed else 'Released', (x, y)))
    id += 1
    single = {
    
    'id': id, 'x': x, 'y': y, "event": "click", "button": str(button),
              'action': 'pressed' if pressed else 'released',
              'time': (datetime.datetime.now() - last_time).total_seconds()}
    last_time = datetime.datetime.now()
    operate_record.append(single)
    make_screenshot(x - 75, y - 50, x + 75, y + 50)


# 监听键盘按键按下
def on_press(key):
    global status_flag
    if not status_flag:
        return status_flag
    # 监听按键
    print('{0} pressed'.format(key))


# 监听按键释放信息
def on_release(key):
    global status_flag
    global operate_record
    # 监听释放
    print('{0} release'.format(key))
    # 若按下的为ESC键，终止程序
    if key == Key.esc:
        with open('./record.txt', 'w') as fp:
            for i in operate_record:
                fp.write(str(i) + '\n')
        print("end")
        status_flag = False
        return status_flag


def listener_mouse():
    with pynput.mouse.Listener(on_move=on_move, on_click=on_click) as listener:
        listener.join()


def listener_key():
    with pynput.keyboard.Listener(on_press=on_press, on_release=on_release)as listener:
        listener.join()


# 一个鼠标监听器是一个线程。线程，所有的回调将从线程调用。从任何地方调用pynput.mouse.Listener.stop，或者调用pynput.mouse.Listener.StopException或从回调中返回False来停止监听器。


if __name__ == '__main__':
    if not os.path.exists('./pic'):
        os.mkdir('./pic')
    t1 = Thread(target=listener_mouse)
    t2 = Thread(target=listener_key)
    t1.start()
    t2.start()

Operation part

# -*- coding:utf-8 -*-


# 控制鼠标
# 读鼠标坐标
import json
import os
import time
from threading import Thread

import cv2
from PIL import ImageGrab
from pynput.keyboard import Key

import pynput

status_flag = True
mouse = pynput.mouse.Controller()
calibration = [0, 0]


def make_screenshot(x1, y1, x2, y2):
    """截图

    :param x1: 开始截图的x1坐标
    :param y1: 开始截图的x1标
    :param x2: 开始截图的x2坐标
    :param y2: 结束截图的y2坐标
    :return: None
    """
    # id: 图片ID
    global id
    bbox = (x1, y1, x2, y2)
    im = ImageGrab.grab(bbox)
    im.save('./temp/present.png')  # 保存截图文件的路径


def identify_pictures(img1Path, img2Path):
    img = cv2.imread(img1Path, 0)
    img2 = img.copy()
    template = cv2.imread(img2Path, 0)
    w, h = template.shape[::-1]

    meth = 'cv2.TM_CCOEFF'
    img = img2.copy()

    method = eval(meth)

    res = cv2.matchTemplate(img, template, method)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
    return max_loc


def setPosition(x, y):
    global mouse
    # 本人电脑点击时鼠标坐标会放大偏移，大概会放大1.25倍
    mouse.position = (x / 1.25, y / 1.25)


def setLeftPress():
    global mouse
    mouse.press(pynput.mouse.Button.left)


def setLeftRelease():
    global mouse
    mouse.release(pynput.mouse.Button.left)


def setRightPress():
    global mouse
    mouse.press(pynput.mouse.Button.right)


def setRightRelease():
    global mouse
    mouse.release(pynput.mouse.Button.right)


def on_press(key):
    # 监听按键
    print('{0} pressed'.format(key))


def on_release(key):
    global status_flag
    # 监听释放
    print('{0} release'.format(key))
    if key == Key.esc:
        status_flag = False
        return status_flag


# 通过查找原始鼠标下的图片位置与现在鼠标下的位置进行对比，计算偏移量
def calibration_offset(i):
    make_screenshot(i['x'] - 150 + calibration[0], i['y'] - 100 + calibration[1], i['x'] + 150 + calibration[0],
                    i['y'] + 100 + calibration[1])
    inner_top_left = identify_pictures('./temp/present.png', './pic/' + str(i['id']) + '.png')
    outside = (
        i['x'] - 150 + inner_top_left[0] + calibration[0], i['y'] - 100 + inner_top_left[1] + calibration[1])

    calibration[0] = outside[0] - (i['x'] - 75)
    calibration[1] = outside[1] - (i['y'] - 50)


def operate_mouse():
    global status_flag
    record = []
    with open("./record.txt", 'r') as fp:
        for line in fp:
            record.append(json.loads(line.replace("\'", "\"")))
    for i in record:
        # print(i)
        if not status_flag:
            break
        if i['event'] == 'move':
            if i['id'] % 100 == 0 and (i['x'] >= 200 or i['x'] <= 1920 - 200) and (i['y'] > 100 or i['y'] < 1080 - 100):
                calibration_offset(i)
            i['x'] = i['x'] + calibration[0]
            i['y'] = i['y'] + calibration[1]
            setPosition(i['x'], i['y'])
        elif i['event'] == 'click':
            calibration_offset(i)
            setPosition(i['x'] + calibration[0], i['y'] + calibration[1])
            if i['action'] == 'pressed':
                if i['button'] == 'Button.left':
                    setLeftPress()
                elif i['button'] == 'Button.right':
                    setRightPress()
            elif i['action'] == 'released':
                if i['button'] == 'Button.left':
                    setLeftRelease()
                elif i['button'] == 'Button.right':
                    setRightRelease()
        time.sleep(i['time'])
    status_flag = False
    print("end")


def listener_key():
    with pynput.keyboard.Listener(on_press=on_press, on_release=on_release)as listener:
        listener.join()


if __name__ == '__main__':
    if not os.path.exists('./temp'):
        os.mkdir('./temp')
    t1 = Thread(target=operate_mouse)
    t2 = Thread(target=listener_key)
    t1.start()
    t2.start()