最近写安卓手机客户端，要和后台通讯， python 写了个后台服务用于自测，感觉代码有通用性，发下吧。
设计：
分成三个部分， 报文设计，后台设计，后台测试用例。后台设计的比较挫，但是可以用。
细节：
报文部分
//包头和包体：sizeof(Pkg)=16+msgLen+extLen
struct Pkg{
struct PkgHdr hdr; //包头，固定长度sizeof(PkgHdr)
uint8_t msg[msgLen]; //放置包体待解析消息,json,pb,tlv等,未使用则为空
uint8_t ext[extLen]; //放置二进制扩展数据(文件或数据流)，未使用则为空
};
//包头和包体：sizeof(Pkg)=16+msgLen+extLen
struct Pkg{
struct PkgHdr hdr; //包头，固定长度sizeof(PkgHdr)
uint8_t msg[msgLen]; //放置包体待解析消息,json,pb,tlv等,未使用则为空
uint8_t ext[extLen]; //放置二进制扩展数据(文件或数据流)，未使用则为空
};
后台模块
# coding=utf-8
import argparse
import logging
import os
import time
import uuid
import json
import threading
import multiprocessing
import random
import select
import socket
import queue
import uuid
import struct
from enum import Enum, unique
import tornado.ioloop
g_select_timeout = 10
class Server(object):
def __init__(self, host='192.168.100.41', port=33333, timeout=2, client_nums=10, speech_recognizer=None):
self.__host = host
self.__port = port
self.__timeout = timeout
self.__client_nums = client_nums
self.__buffer_size = 1024
self.__frame_length = 16
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.setblocking(0)
self.server.settimeout(self.__timeout)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) #keepalive
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) #端口复用
server_host = (self.__host, self.__port)
try:
self.server.bind(server_host)
self.server.listen(self.__client_nums)
except:
raise
#网络相关处理
self.inputs = [self.server]
self.outputs = []
self.message_queues = {}
self.client_info = {}
#业务相关处理
self.store_dir_path = ""
self.filedesc_wavpath_dict = {}
self.speech_recognizer = speech_recognizer
def store_data(self, file_desc, data):
file_desc_list = self.filedesc_wavpath_dict.keys()
if file_desc not in file_desc_list:
random_str_wav = str(_get_ms_time_int_part()) + '-' + str(uuid.uuid4()) + "test.wav"
wav_path = os.path.join(self.store_dir_path, random_str_wav)
self.filedesc_wavpath_dict[file_desc] = wav_path
logging.info("random_str:{}".format(random_str_wav))
logging.info("file desc:{}".format(file_desc))
logging.info("file path:{}".format(wav_path))
with open(self.filedesc_wavpath_dict[file_desc], "ab+") as inputwav:
inputwav.write(data)
return self.filedesc_wavpath_dict[file_desc]
def readMSGHead(self, filedesc):
'''
//数据包头：sizeof(PkgHdr)=16
struct PkgHdr{
uint16_t magic; //magic number:0xfffe
uint16_t bodyLen; //包体长度，即msg和ext总长，不含包头
uint16_t pkgType; //0,请求包 1,返回包 ...后续可扩展
uint16_t cmd; //请求和返回命令字,目前默认填0
uint16_t retCode; //0成功, 其它失败，返回包填写
uint16_t msgFmt; //消息格式0:json 1:protobuf ...
uint16_t msgLen; //消息长度
uint16_t extLen; //扩展包长度
};
'''
frame_length_bytes = filedesc.recv(self.__frame_length)
magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen = struct.unpack('>HHHHHHHH', frame_length_bytes)
return magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen
def logMSG(self, msg ):
msg_head = msg[0:16]
magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen = struct.unpack('>HHHHHHHH', msg_head)
logging.info("========================================")
logging.info("msg_head: {}".format(msg_head))
logging.info("bodyLen: {}".format(bodyLen))
logging.info("pkgType: {}".format(pkgType))
logging.info("cmd: {}".format(cmd))
logging.info("retCode: {}".format(retCode))
logging.info("msgFmt: {}".format(msgFmt))
logging.info("msgLen: {}".format(msgLen))
logging.info("extLen: {}".format(extLen))
logging.info("msgjson: {}".format(msg[16:16+msgLen]))
logging.info("========================================")
def readMSGMsg(self, filedesc, msgLen):
msg = filedesc.recv(msgLen)
msg = msg.decode()
msg_json = json.loads(msg)
return msg_json
def readMSGExt(self, filedesc, extLen):
ext = filedesc.recv(extLen)
return ext
def decodeMSG(self, filedesc):
MSGHeadInfo = self.readMSGHead(filedesc)
logging.info("decodeMSG MSGHeadInfo:{}".format(MSGHeadInfo))
msgLen = MSGHeadInfo[6]
extLen = MSGHeadInfo[7]
logging.info("decodeMSG msgLen:{}".format(msgLen))
logging.info("decodeMSG extLen:{}".format(extLen))
msg_json = self.readMSGMsg(filedesc, msgLen)
Ext = self.readMSGExt(filedesc, extLen)
return msg_json, Ext
def encodeMSG(self, pkgType_input, errCode, errMsg, refresh, text, ext):
magic = 0xfffe
bodyLen = 0
pkgType = pkgType_input
cmd = 0
retCode = 0
msgFmt = 0
msgLen = 0
extLen = 0
msg_json_data = {"errCode": errCode, "errMsg": errMsg, "refresh": refresh, "text": text}
msg = json.dumps(msg_json_data)
if msg:
msgLen = len(msg)
if ext:
extLen = len(ext)
bodyLen = msgLen + extLen
msgHeadByteArray = bytearray(struct.pack('>HHHHHHHH', magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen))
msgMsgByteArray = bytearray(str(msg).encode())
msg = msgHeadByteArray + msgMsgByteArray
if ext:
msgExtByteArray = bytearray(ext)
msg = msg + msgExtByteArray
return msg
def call_asr(self, wav_file_path ):
decode_result = self.speech_recognizer.wav_to_txt(wav_file_path)
return decode_result
def run(self):
while True:
readable , writable , exceptional = select.select(self.inputs, self.outputs, self.inputs, g_select_timeout)
if not (readable or writable or exceptional) :
continue
for s in readable :
if s is self.server:#是客户端连接
connection, client_address = s.accept()
#print "connection", connection
print( "%s connect. " %str(client_address) )
connection.setblocking(False)
self.inputs.append(connection) #客户端添加到inputs
self.client_info[connection] = str(client_address)
self.message_queues[connection] = queue.Queue() #每个客户端一个消息队列
else:#是client, 数据发送过来
receiveMSG = None
try:
receiveMSG = self.decodeMSG(s)
except Exception as e:
err_msg = "Client Error!!!"
logging.error(err_msg)
logging.error(str(e))
if receiveMSG :
msg_json, Ext = receiveMSG
logging.info("receive msg_json: {}".format(msg_json))
logging.info("receiveExt length: {}".format(len(Ext)))
wav_file_path = self.store_data(s, Ext)
dataoutput = "%s %s " % (time.strftime("%Y-%m-%d %H:%M:%S"), self.client_info[s])
#dataoutput = "message from server"
self.message_queues[s].put(dataoutput)
if s not in self.outputs:
self.outputs.append(s)
else:
#Interpret empty result as closed connection
print ("Client:%s Close." % str( self.client_info[s]) )
if s in self.outputs :
self.outputs.remove(s)
self.inputs.remove(s)
s.close()
del self.message_queues[s]
del self.client_info[s]
if s in self.filedesc_wavpath_dict.keys():
del self.filedesc_wavpath_dict[s]
for s in writable: #outputs 有消息就要发出去了
try:
next_msg = self.message_queues[s].get_nowait() #非阻塞获取
except queue.Empty:
err_msg = "Output Queue is Empty!"
#g_logFd.writeFormatMsg(g_logFd.LEVEL_INFO, err_msg)
self.outputs.remove(s)
except Exception as e: #发送的时候客户端关闭了则会出现writable和readable同时有数据，会出现message_queues的keyerror
err_msg = "Send Data Error! ErrMsg:%s" % str(e)
logging.error(err_msg)
if s in self.outputs:
self.outputs.remove(s)
else:
try:
cli = s
pkgType_input = 1
errCode = 0
errMsg = "OK"
refresh = 1
text = next_msg
ext = None
'''
logging.info("errCode :{}".format(errCode))
logging.info("errMsg :{}".format(errMsg))
logging.info("refresh :{}".format(refresh))
logging.info("text :{}".format(text))
logging.info("ext :{}".format(ext))
'''
msgresp = self.encodeMSG(pkgType_input, errCode, errMsg, refresh, text, ext)
self.logMSG(msgresp)
cli.send(msgresp)
except Exception as e: #发送失败就关掉
err_msg = "Send Data to %s Error! ErrMsg:%s" % (str(self.client_info[cli]), str(e))
logging.error(err_msg)
print( "Client: %s Close Error." % str(self.client_info[cli]) )
if cli in self.inputs:
self.inputs.remove(cli)
cli.close()
if cli in self.outputs:
self.outputs.remove(s)
if cli in self.message_queues:
del self.message_queues[s]
del self.client_info[cli]
del self.filedesc_wavpath_dict[s]
for s in exceptional:
logging.error("Client:%s Close Error." % str(self.client_info[cli]))
if s in self.inputs:
self.inputs.remove(s)
s.close()
if s in self.outputs:
self.outputs.remove(s)
if s in self.message_queues:
del self.message_queues[s]
del self.client_info[s]
del self.filedesc_wavpath_dict[s]
if "__main__" == __name__:
logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s",
filename='realtime_asr_server.log',
level=logging.INFO)
Server().run()
后台测试代码
import sys
import time
import socket
import threading
import logging
import json
import struct
class Client(object):
def __init__(self, host, port=33333, timeout=1, reconnect=2):
self.__host = host
self.__port = port
self.__timeout = timeout
self.__buffer_size = 1024
self.__flag = 1
self.client = None
self.__lock = threading.Lock()
self.__frame_length = 16
def readMSGHead(self, filedesc):
'''
//数据包头：sizeof(PkgHdr)=16
struct PkgHdr{
uint16_t magic; //magic number:0xfffe
uint16_t bodyLen; //包体长度，即msg和ext总长，不含包头
uint16_t pkgType; //0,请求包 1,返回包 ...后续可扩展
uint16_t cmd; //请求和返回命令字,目前默认填0
uint16_t retCode; //0成功, 其它失败，返回包填写
uint16_t msgFmt; //消息格式0:json 1:protobuf ...
uint16_t msgLen; //消息长度
uint16_t extLen; //扩展包长度
};
'''
frame_length_bytes = filedesc.recv(self.__frame_length)
magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen = struct.unpack('>HHHHHHHH', frame_length_bytes)
return magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen
def readMSGMsg(self, filedesc, msgLen):
msg = filedesc.recv(msgLen)
msg = msg.decode()
msg_json = json.loads(msg)
return msg_json
def readMSGExt(self, filedesc, extLen):
ext = filedesc.recv(extLen)
return ext
def decodeMSG(self, filedesc):
MSGHeadInfo = self.readMSGHead(filedesc)
msgLen = MSGHeadInfo[6]
extLen = MSGHeadInfo[7]
logging.info("msgLen:{}".format(msgLen))
logging.info("extLen:{}".format(extLen))
msg_json = self.readMSGMsg(filedesc, msgLen)
Ext = self.readMSGExt(filedesc, extLen)
return msg_json, Ext
def encodeMSG(self, pkgType_input, errCode, errMsg, refresh, text, ext):
magic = 0xfffe
bodyLen = 0
pkgType = pkgType_input
cmd = 0
retCode = 0
msgFmt = 0
msgLen = 0
extLen = 0
msg_json_data = {"errCode": errCode, "errMsg": errMsg, "refresh": refresh, "text": text}
msg = json.dumps(msg_json_data)
if msg:
msgLen = len(msg)
if ext:
extLen = len(ext)
bodyLen = msgLen + extLen
msgHeadByteArray = bytearray(
struct.pack('>HHHHHHHH', magic, bodyLen, pkgType, cmd, retCode, msgFmt, msgLen, extLen))
msgMsgByteArray = bytearray(str(msg).encode())
msg = msgHeadByteArray + msgMsgByteArray
if ext:
msgExtByteArray = bytearray(ext)
msg = msg + msgExtByteArray
return msg
@property
def flag(self):
return self.__flag
@flag.setter
def flag(self, new_num):
self.__flag = new_num
def __connect(self):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# client.bind(('0.0.0.0', 12345,))
client.setblocking(True)
client.settimeout(self.__timeout)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # 端口复用
server_host = (self.__host, self.__port)
try:
client.connect(server_host)
except:
raise
return client
def send_msg(self):
if not self.client:
return
while True:
time.sleep(0.1)
#data = sys.stdin.readline().strip()
#data = input("input string:")
data = "hello from client"
if "exit" == data.lower():
with self.__lock:
self.flag = 0
break
data = self.encodeMSG(0, 0, "OK", 0, data, b"xiaojiba")
logging.info("client sendall")
self.client.sendall(data)
return
def recv_msg(self):
if not self.client:
return
while True:
data = None
with self.__lock:
if not self.flag:
print('ByeBye~~')
break
try:
logging.info("client recv")
msg_json, Ext = self.decodeMSG(self.client)
text = msg_json['text']
isend = msg_json['refresh']
logging.info("text:{}".format(text))
logging.info("isend:{}".format(isend))
except socket.timeout:
continue
except:
raise
if data:
print("%s\n" % text)
time.sleep(0.1)
return
def run(self):
self.client = self.__connect()
send_proc = threading.Thread(target=self.send_msg)
recv_proc = threading.Thread(target=self.recv_msg)
recv_proc.start()
send_proc.start()
recv_proc.join()
send_proc.join()
self.client.close()
if "__main__" == __name__:
logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s",
#filename='realtime_asr_server.log',
level=logging.INFO)
Client('192.168.100.41').run()