OpenCV+Mediapipe+UDP+Unity挥手电子书翻页
效果视频如下
OpenCV+Mediapipe+UDP+Unity挥手翻页
Python端
其实很简单,基本都是现成的东西。
主要做的事情:
- 读取XML配置文件,主要是摄像头的配置(包括分辨率、帧率)、网络配置、调试信息的配置等。
- Opencv读取摄像头,Mediapipe识别手,UDP发送识别到的手坐标数据。
- PyInstaller将py代码打包为EXE。
这里有个坑:
pyinstaller打包的exe,总提示找不到mediapipe路径,需要打包时手工加入mediapipe库:pyinstaller --add-data="E:/MyPythonProjects/HandTouch/venv/Lib/site-packages/mediapipe/modules;mediapipe/modules" -w main.py
第二个坑:打包完的Exe路径不能含有中文字符,否则还是提示找不到路径。
关键代码段也就这么些:
cap = cv2.VideoCapture(usbpt)
if width > 0 and height > 0:
cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
if vfps > 0:
cap.set(cv2.CAP_PROP_FPS, vfps)
mpHands = mp.solutions.hands
hands = mpHands.Hands(max_num_hands=1)
mpDraw = mp.solutions.drawing_utils
pTime = 0
while True:
success, img = cap.read()
if success:
if bFilp:
img = cv2.flip(img, 1)
imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h, w, t = img.shape
result = hands.process(imgRGB)
if result.multi_hand_landmarks:
lm = result.multi_hand_landmarks[0]
hd = lm.landmark[0]
x = int(hd.x * w)
y = int(hd.y * h)
if bShowHand:
mpDraw.draw_landmarks(img, lm, mpHands.HAND_CONNECTIONS)
cv2.circle(img, (x, y), 3, (0, 255, 255), 3)
skt.sendto(('[%d,%d]' % (x, y)).encode('utf-8'), (host, port))
if bShowFps:
cTime = time.time()
fps = int(1 / (cTime - pTime))
pTime = cTime
cv2.putText(img, str(int(fps)), (20, 60), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 255), 3)
if bShowImg:
cv2.imshow("Image", img)
if bEscQuit and (cv2.waitKey(1) & 0xff == 27):
break
else:
cv2.waitKey(1)
cv2.destroyAllWindows()
cap.release()
Unity端
数据收到后,有穷自动状态机,解析一下数据:
// 数据接收线程
private void ReceiveFrom()
{
EndPoint remote = new IPEndPoint(IPAddress.Any, 0);
byte[] data = new byte[1024];
while (IsStart)
{
try
{
int len = m_socket.ReceiveFrom(data, data.Length, SocketFlags.None, ref remote);
for (int i = 0; i < len; ++i)
{
switch (m_state)
{
case ReceState.START:
if (data[i] == '[')
{
m_state = ReceState.XDATA;
m_pos.x = 0;
}
break;
case ReceState.XDATA:
if (data[i] >= '0' && data[i] <= '9')
m_pos.x = m_pos.x * 10 + (data[i] - '0');
else if (data[i] == ',')
{
m_state = ReceState.YDATA;
m_pos.y = 0;
}
else
m_state = ReceState.START;
break;
case ReceState.YDATA:
if (data[i] >= '0' && data[i] <= '9')
m_pos.y = m_pos.y * 10 + (data[i] - '0');
else if (data[i] == ']')
{
thePosition.Enqueue(m_pos);
m_state = ReceState.START;
}
else
m_state = ReceState.START;
break;
}
}
}
catch {
}
}
}
对于收到的数据,进一步解析,获得手势:
这里只是用了很简单的判断方式:
- 只判断水平方向的挥手
- 只看累计位移量(像素)是否达到阈值
private void OnReceivePos(Vector2 pos)
{
if( bCanTrigger )
{
waitTime = 0.333f;
if (posCount++ > 0)
{
float delta = pos.x - lastPos;
totalDelta += delta;
if (posCount > 5 )
{
if (totalDelta > flipValue)
{
bCanTrigger = false;
interval = autoFlip.PageFlipTime;
lastPos = 0;
totalDelta = 0;
posCount = 0;
PrevPage();
}
else if (totalDelta < -flipValue)
{
bCanTrigger = false;
interval = autoFlip.PageFlipTime;
lastPos = 0;
totalDelta = 0;
posCount = 0;
NextPage();
}
}
}
lastPos = pos.x;
}
}