Python uiautomator2 基于图像识别操作的核心代码封装类
方案1
以下是基于图像识别操作的uiautomator2核心代码封装类，包含函数实例，并加上中文注释。该类文件可直接使用
import time
import os
import re
import random
import json
import logging
import subprocess
import functools
import collections
import traceback
import xml.etree.ElementTree as ET
from typing import List, Dict, Union, Tuple
import uiautomator2 as u2
import cv2
import numpy as np
from PIL import Image
# 定义一个Logger对象，用于记录日志
logger = logging.getLogger(__name__)
# 定义一个装饰器函数，用于记录函数执行时间
def timeit(fn):
@functools.wraps(fn)
def wrapper(*args, **kwargs):
start = time.time()
ret = fn(*args, **kwargs)
end = time.time()
logger.debug(f"{fn.__name__} elapsed time: {end - start:.3f}s")
return ret
return wrapper
class UIAutomator2:
def __init__(self, device: str = None, pkg_name: str = None):
"""
初始化uiautomator2对象
:param device: 设备ID或IP地址，如果为None则默认连接所有设备
:param pkg_name: 待测试应用的包名
"""
self.d = u2.connect(device)
self.pkg_name = pkg_name
self.width, self.height = self.d.window_size()
@timeit
def click(self, x: int, y: int, duration: float = 0.1):
"""
点击屏幕上的一个点
:param x: 点击点的X坐标
:param y: 点击点的Y坐标
:param duration: 点击的持续时间
"""
self.d.click(x, y, duration=duration)
@timeit
def long_click(self, x: int, y: int, duration: float = 0.5):
"""
长按屏幕上的一个点
:param x: 长按点的X坐标
:param y: 长按点的Y坐标
:param duration: 长按的持续时间
"""
self.d.long_click(x, y, duration=duration)
@timeit
def swipe(self, x1: int, y1: int, x2: int, y2: int, duration: float = 0.5):
"""
在屏幕上滑动
:param x1: 起始点的X坐标
:param y1: 起始点的Y坐标
:param x2: 终止点的X坐标
:param y2: 终止点的Y坐标
:param duration: 滑动的持续时间
"""
self.d.swipe(x1, y1, x2, y2, duration=duration)
@timeit
def press_keycode(self, keycode: int):
"""
按下一个键
:param keycode: 键值
"""
self.d.press(keycode)
@timeit
def back(self):
"""
模拟按下返回键
"""
self.d.press("back")
@timeit
def home(self):
"""
模拟按下Home键
"""
self.d.press("home")
@timeit
def recent(self):
"""
模拟按下最近任务键
"""
self.d.press("recent")
@timeit
def start_app(self, activity: str = None):
"""
启动应用
:param activity: 应用的主Activity名称
"""
if activity:
self.d.app_start(self.pkg_name, activity)
else:
self.d.app_start(self.pkg_name)
@timeit
def stop_app(self):
"""
停止应用
"""
self.d.app_stop(self.pkg_name)
@timeit
def clear_app_data(self):
"""
清除应用数据
"""
self.d.app_clear(self.pkg_name)
@timeit
def get_current_activity(self) -> str:
"""
获取当前Activity名称
:return: 当前Activity名称
"""
return self.d.current_app().get("activity")
@timeit
def get_current_package(self) -> str:
"""
获取当前应用的包名
:return: 当前应用的包名
"""
return self.d.current_app().get("package")
@timeit
def get_device_info(self) -> Dict[str, Union[str, int]]:
"""
获取设备信息
:return: 设备信息字典
"""
info = self.d.info
return {
"brand": info["brand"],
"model": info["model"],
"device_id": info["deviceid"],
"sdk_version": info["sdkInt"],
"screen_width": self.width,
"screen_height": self.height,
}
@timeit
def screenshot(self, filename: str = None) -> Union[None, np.ndarray]:
"""
截图并返回截图的numpy数组
:param filename: 保存截图的文件名，如果为None则不保存
:return: 截图的numpy数组
"""
img = self.d.screenshot()
if filename:
img.save(filename)
return np.array(img)
@timeit
def find_image(self, template: str, threshold: float = 0.8) -> Union[Tuple[int, int], None]:
"""
在当前屏幕上查找指定图片的位置
:param template: 模板图片的文件名或numpy数组
:param threshold: 匹配阈值，取值范围为0到1之间，值越大匹配越严格
:return: 如果找到则返回图片的左上角坐标，否则返回None
"""
if isinstance(template, str):
img = cv2.imread(template, cv2.IMREAD_GRAYSCALE)
else:
img = template
res = cv2.matchTemplate(self.screenshot(), img, cv2.TM_CCOEFF_NORMED)
loc = np.where(res >= threshold)
if len(loc[0]) > 0:
return loc[1][0], loc[0][0]
return None
@timeit
def click_image(self, template: str, threshold: float = 0.8, duration: float = 0.1):
"""
在当前屏幕上点击指定图片的位置
:param template: 模板图片的文件名或numpy数组
:param threshold: 匹配阈值，取值范围为0到1之间，值越大匹配越严格
:param duration: 点击的持续时间
"""
pos = self.find_image(template, threshold)
if pos:
self.click(*pos, duration=duration)
@timeit
def wait_image(self, template: str, timeout: float = 10.0, threshold: float = 0.8) -> Union[Tuple[int, int], None]:
"""
等待指定图片出现
:param template: 模板图片的文件名或numpy数组
:param timeout: 超时时间，单位为秒
:param threshold: 匹配阈值，取值范围为0到1之间，值越大匹配越严格
:return: 如果找到则返回图片的左上角坐标，否则返回None
"""
start = time.time()
while time.time() - start < timeout:
pos = self.find_image(template, threshold)
if pos:
return pos
time.sleep(0.2)
return None
@timeit
def wait_click_image(self, template: str, timeout: float = 10.0, threshold: float = 0.8, duration: float = 0.1):
"""
等待并点击指定图片
:param template: 模板图片的文件名或numpy数组
:param timeout: 超时时间，单位为秒
:param threshold: 匹配阈值，取值范围为0到1之间，值越大匹配越严格
:param duration: 点击的持续时间
"""
pos = self.wait_image(template, timeout, threshold)
if pos:
self.click(*pos, duration=duration)
方案2
由于Python uiautomator2库提供了非常强大的UI自动化测试功能，但是很多人对其使用还是不太熟悉，因此本文将介绍一个基于图像识别操作的核心代码封装类，该类包含了多个函数实例，并且每个函数都有中文注释，方便大家快速上手使用。
# coding:utf-8
import uiautomator2 as u2
import time
import os
import cv2
import numpy as np
class UiImageAutomator(object):
"""
基于图像识别操作的封装类
"""
def __init__(self, device_sn):
"""
初始化函数
:param device_sn: 设备序列号
"""
self.d = u2.connect(device_sn)
self.width, self.height = self.d.window_size()
def click_image(self, image_path, timeout=10):
"""
点击指定图片
:param image_path: 图片路径
:param timeout: 超时时间（秒），默认为10秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
try:
img = cv2.imread(image_path, 0)
img_width, img_height = img.shape[::-1]
screen = self.d.screenshot(format='opencv')
result = cv2.matchTemplate(screen, img, cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
if max_val > 0.8:
x, y = max_loc[0] + img_width / 2, max_loc[1] + img_height / 2
self.d.click(x / self.width, y / self.height)
return True
except Exception as e:
print(e)
time.sleep(1)
def click_image_until(self, image_path, until_image_path, timeout=30):
"""
点击指定图片，直到出现目标图片
:param image_path: 图片路径
:param until_image_path: 目标图片路径
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if self.is_image_exist(until_image_path):
return True
self.click_image(image_path, 1)
def click_image_times(self, image_path, times=1):
"""
点击指定图片，指定次数
:param image_path: 图片路径
:param times: 点击次数，默认为1次
:return: True/False
"""
for i in range(times):
if not self.click_image(image_path):
return False
return True
def click_image_until_gone(self, image_path, timeout=30):
"""
点击指定图片，直到该图片消失
:param image_path: 图片路径
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if not self.is_image_exist(image_path):
return True
self.click_image(image_path, 1)
def click_image_until_color(self, image_path, color, threshold=10, timeout=30):
"""
点击指定图片，直到该图片上某一像素点的颜色与指定颜色相似
:param image_path: 图片路径
:param color: 指定颜色，格式为(B, G, R)
:param threshold: 相似度阈值，默认为10
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
try:
img = cv2.imread(image_path)
h, w, _ = img.shape
center_color = img[h // 2, w // 2]
if abs(center_color[0] - color[0]) <= threshold and abs(center_color[1] - color[1]) <= threshold and abs(center_color[2] - color[2]) <= threshold:
self.click_image(image_path, 1)
return True
except Exception as e:
print(e)
time.sleep(1)
def click_image_until_color_gone(self, image_path, color, threshold=10, timeout=30):
"""
点击指定图片，直到该图片上某一像素点的颜色与指定颜色不再相似
:param image_path: 图片路径
:param color: 指定颜色，格式为(B, G, R)
:param threshold: 相似度阈值，默认为10
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
try:
img = cv2.imread(image_path)
h, w, _ = img.shape
center_color = img[h // 2, w // 2]
if abs(center_color[0] - color[0]) > threshold or abs(center_color[1] - color[1]) > threshold or abs(center_color[2] - color[2]) > threshold:
return True
except Exception as e:
print(e)
self.click_image(image_path, 1)
def click_image_until_text(self, image_path, text, threshold=0.7, timeout=30):
"""
点击指定图片，直到该图片上出现指定文本
:param image_path: 图片路径
:param text: 指定文本
:param threshold: 相似度阈值，默认为0.7
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if self.is_text_exist(image_path, text, threshold):
self.click_image(image_path, 1)
return True
time.sleep(1)
def click_image_until_text_gone(self, image_path, text, threshold=0.7, timeout=30):
"""
点击指定图片，直到该图片上不再出现指定文本
:param image_path: 图片路径
:param text: 指定文本
:param threshold: 相似度阈值，默认为0.7
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if not self.is_text_exist(image_path, text, threshold):
return True
self.click_image(image_path, 1)
def click_text(self, text, timeout=10):
"""
点击指定文本
:param text: 指定文本
:param timeout: 超时时间（秒），默认为10秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
try:
self.d(text=text).click()
return True
except Exception as e:
print(e)
time.sleep(1)
def click_text_until(self, text, until_image_path, timeout=30):
"""
点击指定文本，直到出现目标图片
:param text: 指定文本
:param until_image_path: 目标图片路径
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if self.is_image_exist(until_image_path):
return True
self.click_text(text, 1)
def click_text_times(self, text, times=1):
"""
点击指定文本，指定次数
:param text: 指定文本
:param times: 点击次数，默认为1次
:return: True/False
"""
for i in range(times):
if not self.click_text(text):
return False
return True
def click_text_until_gone(self, text, timeout=30):
"""
点击指定文本，直到该文本消失
:param text: 指定文本
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if not self.is_text_exist(text):
return True
self.click_text(text, 1)
def click_text_until_color(self, text, color, threshold=10, timeout=30):
"""
点击指定文本，直到该文本上某一像素点的颜色与指定颜色相似
:param text: 指定文本
:param color: 指定颜色，格式为(B, G, R)
:param threshold: 相似度阈值，默认为10
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
try:
center_color = self.get_text_center_color(text)
if abs(center_color[0] - color[0]) <= threshold and abs(center_color[1] - color[1]) <= threshold and abs(center_color[2] - color[2]) <= threshold:
self.click_text(text, 1)
return True
except Exception as e:
print(e)
time.sleep(1)
def click_text_until_color_gone(self, text, color, threshold=10, timeout=30):
"""
点击指定文本，直到该文本上某一像素点的颜色与指定颜色不再相似
:param text: 指定文本
:param color: 指定颜色，格式为(B, G, R)
:param threshold: 相似度阈值，默认为10
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
try:
center_color = self.get_text_center_color(text)
if abs(center_color[0] - color[0]) > threshold or abs(center_color[1] - color[1]) > threshold or abs(center_color[2] - color[2]) > threshold:
return True
except Exception as e:
print(e)
self.click_text(text, 1)
def click_text_until_text(self, text, until_text, threshold=0.7, timeout=30):
"""
点击指定文本，直到该文本下出现指定文本
:param text: 指定文本
:param until_text: 目标文本
:param threshold: 相似度阈值，默认为0.7
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if self.is_text_exist(text) and self.is_text_exist(text, until_text, threshold):
self.click_text(text, 1)
return True
time.sleep(1)
def click_text_until_text_gone(self, text, until_text, threshold=0.7, timeout=30):
"""
点击指定文本，直到该文本下不再出现指定文本
:param text: 指定文本
:param until_text: 目标文本
:param threshold: 相似度阈值，默认为0.7
:param timeout: 超时时间（秒），默认为30秒
:return: True/False
"""
start_time = time.time()
while True:
current_time = time.time()
if current_time - start_time > timeout:
print("Timeout")
return False
if not self.is_text_exist(text, until_text, threshold):
return True
self.click_text(text, 1)
def get_text_center_color(self, text):
"""
获取指定文本中心像素点颜色
:param text: 指定文本
:return: 颜色，格式为(B, G, R)
"""
bounds = self.d(text=text).info['bounds']
x = (bounds['left'] + bounds['right']) / 2
y = (bounds['top'] + bounds['bottom']) / 2
screen = self.d.screenshot(format='opencv')
return screen[int(y), int(x)]
def is_image_exist(self, image_path, threshold=0.8):
"""
判断指定图片是否存在
:param image_path: 图片路径
:param threshold: 相似度阈值，默认为0.8
:return: True/False
"""
try:
img = cv2.imread(image_path, 0)
img_width, img_height = img.shape[::-1]
screen = self.d.screenshot(format='opencv')
result = cv2.matchTemplate(screen, img, cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
if max_val > threshold:
return True
except Exception as e:
print(e)
return False
def is_text_exist(self, text, other_text=None, threshold=0.7):
"""
判断指定文本是否存在
:param text: 指定文本
:param other_text: 其他文本，用于判断指定文本下是否出现了目标文本，默认为None
:param threshold: 相似度阈值，默认为0.7
:return: True/False
"""
try:
if other_text is None:
self.d(text=text)
return True
else:
self.d(text=text).down(text=other_text)
return True
except Exception as e:
print(e)
return False
def long_click_image(self, image_path, duration=1):
"""
长按指定图片
:param image_path: 图片路径
:param duration: 长按时间（秒），默认为1秒
:return: True/False
"""
try:
img = cv2.imread(image_path, 0)
img_width, img_height = img.shape[::-1]
screen = self.d.screenshot(format='opencv')
result = cv2.matchTemplate(screen, img, cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
if max_val > 0.8:
x, y = max_loc[0] + img_width / 2, max_loc[1] + img_height / 2
self.d.long_click(x / self.width, y / self.height, duration)
return True
except Exception as e:
print(e)
return False
def long_click_text(self, text, duration=1):
"""
长按指定文本
:param text: 指定文本
:param duration: 长按时间（秒），默认为1秒
:return: True/False
"""
try:
self.d(text=text).long_click(duration=duration)
return True
except Exception as e:
print(e)
return False