Unity and Python communicate with each other through sockets to send custom data packets. This is a way to use Unity to build scenes and do data processing through Python, which can effectively take advantage of the advantages of two different languages.
I have encapsulated the corresponding operation into a corresponding module, SocketTools.cs. Let's take a look at the specific code usage first (this code is a simple code that encodes the image rendered by the unity main camera and sends it to python through the socket. And receive a code that Python processes the image results and displays them):
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.Networking;
using System.Text;
using System.Runtime.InteropServices;
using System.Linq;
using UnityEngine.UI;
public class HttpClient : MonoBehaviour
{
public Camera cam;
//关于相机的参数
public RawImage rawImage;
private float timeOld = 0;
private RenderTexture cameraView = null;
private Texture2D screenShot = null;
private Texture2D texture = null;
private byte[] image_recv_bytes = new byte[0];
SocketTools tools = new SocketTools();
private bool renderState = false;
// Start is called before the first frame update
void Start()
{
cameraView = new RenderTexture(Screen.width, Screen.height, 8);
cameraView.enableRandomWrite = true;
texture = new Texture2D(500, 400);
Rigidbody rigidbody = GetComponent<Rigidbody>();
rigidbody.angularVelocity = new Vector3(0,0.2f,0);
timeOld = Time.time;
tools.SocketInit();
tools.SetCallBackFun(CallBackFun);
tools.ConnectServer("127.0.0.1", 4444);
}
public void CallBackFun(SocketStatus socketStatus, System.Object obj)
{
switch (socketStatus)
{
case SocketStatus.eSocketConnecting:
Debug.Log("正在连接服务器...");
break;
case SocketStatus.eSocketConnected:
Debug.Log("服务器连接成功...");
break;
case SocketStatus.eSocketReceived:
{
//Debug.Log("接收到消息：" + System.Text.Encoding.Default.GetString((byte[])obj));
image_recv_bytes = (byte[])obj;
}
break;
// case SocketStatus.eSocketSending:
// Debug.Log("正在发送消息...");
// break;
// case SocketStatus.eSocketSent:
// Debug.Log("消息已发送...");
// break;
case SocketStatus.eSocketClosed:
Debug.Log("服务器已断开...");
break;
default:
break;
}
}
// Update is called once per frame
void Update()
{
if (tools.Connected && renderState)
{
renderState = false;
if (null == screenShot)
{
screenShot = new Texture2D(Screen.width, Screen.height, TextureFormat.RGB24, false);
}
// 读取屏幕像素进行渲染
cam.targetTexture = cameraView;
cam.Render();
RenderTexture.active = cameraView;
screenShot.ReadPixels(new Rect(0, 0, cameraView.width, cameraView.height), 0, 0);
screenShot.Apply();
byte[] bytes = screenShot.EncodeToJPG();
// //实例化一个文件流--->与写入文件相关联
// FileStream fs = new FileStream("test_unity.jpg", FileMode.Create);
// //开始写入
// fs.Write(bytes, 0, bytes.Length);
// //清空缓冲区、关闭流
// fs.Flush();
// fs.Close();
// return;
cam.targetTexture = null;
try
{
timeOld = Time.time;
tools.SendToServer(tools.PackData(bytes, 1));
}
catch
{
if (!tools.Connected)
{
tools.SocketClose();
Debug.Log("the server has been disconnected. ");
}
}
}
if (image_recv_bytes != null)
{
Debug.Log("图片大小：" + image_recv_bytes.Length + "bytes, 帧间延时：" + Time.deltaTime * 1000 + "ms, 网络时间延迟：" + ((Time.time - timeOld) * 1000).ToString() + "ms");
texture.LoadImage(image_recv_bytes);
image_recv_bytes = null;
rawImage.texture = texture;
renderState = true;
}
if (Input.GetKeyDown(KeyCode.Space))
{
if (tools.Connected)
{
tools.SocketClose();
Debug.Log("socket closed. ");
}
else
{
tools.ConnectServer("127.0.0.1", 4444);
}
}
}
}
The following is the corresponding code effect:
unity+python
Some technical details will be explained in detail below. If you are not interested in the details of this implementation, you can also directly download the project code I have completed:
Next is the detailed technical implementation process. Generally speaking, the main technical means are divided into three parts:
1. Implementation of asynchronous socket in python and unity
2. Implementation of splitting and merging custom data packets in python and unity
3. Implementation of packet authentication for socket communication between python and unity
4. Unity data processing asynchronous encapsulation logic
1. Implementation of asynchronous socket in python and unity
a. python asynchronous socket
Since python here is mainly used as a server, the python code here is mainly implemented by using python as a socket tcp server.
First is the initialization server code
# 初始化服务器
def socket_init(self, host, port):
# 1 创建服务端套接字对象
self.tcp_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# signal_update.emit(0.2, 1, 1, 'socket object created...')
# 设置端口复用，使程序退出后端口马上释放
self.tcp_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
# 2 绑定端口
self.tcp_server.bind((host, port))
# signal_update.emit(0.7, 1, 1, "socket bind successfully...")
print('server port bind successfully, server host: {}, server port: {}...'.format(host, port))
Then start the server monitoring code. First start a new thread to monitor the status of the tcp server, and then implement the corresponding code in the start_thread_fun thread function, as follows:
# 开始的线程函数，外部最好调用 start() 函数，不要调用此函数
# 否则会阻塞
def start_thread_fun(self):
print("server_thread started. ")
# 3 设置监听
self.tcp_server.listen(self.max_connections)
print('start to listen connections from client, max client count: {}'.format(
self.max_connections))
# 4 循环等待客户端连接请求（也就是最多可以同时有128个用户连接到服务器进行通信）
while True:
tcp_client_1, tcp_client_address = self.tcp_server.accept()
self.tcp_clients.append(tcp_client_1)
# 创建多线程对象
thd = threading.Thread(target=self.client_process,
args=(tcp_client_1, tcp_client_address), daemon=True)
# 设置守护主线程 即如果主线程结束了 那子线程中也都销毁了 防止主线程无法退出
thd.setDaemon(True)
# 启动子线程对象
thd.start()
print("new client connected, client address: {}, total client count: {}".format(tcp_client_address, 1))
# 启动服务器
def start(self):
self.start_thread = threading.Thread(target=self.start_thread_fun, daemon=True)
self.start_thread.start()
print("starting server_thread...")Note: The main thread cannot be terminated here, otherwise all sub-threads will stop. You can add time.sleep(100) to the main thread to manually delay the main thread running time.
b. unity asynchronous socket
Unity's asynchronous socket is a bit more troublesome in comparison, because its asynchronous status needs to be manually reset and adjusted through code every time. The process will be explained in detail below.
Regarding the entire process, I think it would be more appropriate to use a flow chart. Here is a simple flow chart of the code:
The code content in each place is as follows:
public void SocketInit()
{
socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
}
public void ConnectServer(string host, int port)
{
try
{
SocketAsyncEventArgs args = new SocketAsyncEventArgs();//创建连接参数对象
this.endPoint = new IPEndPoint(IPAddress.Parse(host), port);
args.RemoteEndPoint = this.endPoint;
args.Completed += OnConnectedCompleted;//添加连接创建成功监听
socket.ConnectAsync(args); //异步创建连接
}
catch (Exception e)
{
Debug.Log("服务器连接异常:" + e);
}
}
private void OnConnectedCompleted(object sender, SocketAsyncEventArgs args)
{
try
{ ///连接创建成功监听处理
if (args.SocketError == SocketError.Success)
{
StartReceiveMessage(); //启动接收消息
}
}
catch (Exception e)
{
Debug.Log("开启接收数据异常" + e);
}
}
private void StartReceiveMessage(int bufferSize = 40960)
{
//启动接收消息
SocketAsyncEventArgs receiveArgs = new SocketAsyncEventArgs();
//设置接收消息的缓存大小，正式项目中可以放在配置 文件中
byte[] buffer = new byte[bufferSize];
//设置接收缓存
receiveArgs.SetBuffer(buffer, 0, buffer.Length);
receiveArgs.RemoteEndPoint = this.endPoint;
receiveArgs.Completed += OnReceiveCompleted; //接收成功
socket.ReceiveAsync(receiveArgs);//开始异步接收监听
}
public void OnReceiveCompleted(object sender, SocketAsyncEventArgs args)
{
try
{
//Debug.Log("网络接收成功线程：" + Thread.CurrentThread.ManagedThreadId.ToString());
if (args.SocketError == SocketError.Success && args.BytesTransferred > 0)
{
//创建读取数据的缓存
byte[] bytes = new byte[args.BytesTransferred];
//将数据复制到缓存中
Buffer.BlockCopy(args.Buffer, 0, bytes, 0, args.BytesTransferred);
Debug.Log(bytes);
//再次启动接收数据监听，接收下次的数据。
StartReceiveMessage();
}
}
catch (Exception e)
{
Debug.Log("接收数据异常：" + e);
}
}
public void SendToServer(byte[] data)
{
try
{
//创建发送参数
SocketAsyncEventArgs sendEventArgs = new SocketAsyncEventArgs();
sendEventArgs.RemoteEndPoint = endPoint;
//设置要发送的数据
sendEventArgs.SetBuffer(data, 0, data.Length);
sendEventArgs.Completed += OnSendCompleted;
//异步发送数据
socket.SendAsync(sendEventArgs);
}
catch (Exception e)
{
Debug.Log("发送数据异常：" + e);
}
}
public void OnSendCompleted(object sender, SocketAsyncEventArgs args)
{
if (args.SocketError == SocketError.Success)
{
Debug.Log("send ok");
}
}
public void SocketClose()
{
socket.Close();
}
The code contains some of the simplest asynchronous c# socket code.
2. Implementation of splitting and merging custom data packets in python and unity
But everyone knows that TCP is a streaming transmission protocol. Especially for large data packets such as images, it cannot be well split between frames. We need to customize the data packet header and data packet body of the protocol. To achieve variable-length data transmission, let’s talk about how to split and combine data at the byte level in python and c#.
a. python
Python mainly uses the struct.pack and struct.unpack functions to split and combine data. The specific code is shown below:
# 判断当前的数据头对不对，如果正确返回解析结果
def process_protocol(self, header):
header_unpack = struct.unpack(self.header_format, header)
if header_unpack[0] == self.header_bytes:
return True, header_unpack
else:
return False, None
def pack_data(self, data, data_type):
if type(data) is str:
data = data.encode()
data_pack = struct.pack(self.header_format, self.header_bytes, data_type, len(data))
# print("datalen:{}".format(len(data)))
return data_pack+data
# 部分相关变量定义
self.header_bytes = b'\x0a\x0b' # 占用两个字节
self.header_format = "2ssi"
b. unity
In Unity's C#, in order to facilitate the analysis of the contents of the corresponding data frame header, the method used is to directly convert the corresponding byte[] type variable into a struct. The corresponding code for converting the two to each other is as follows:
public static object BytesToStruct(byte[] buf, int len, Type type)
{
object rtn;
IntPtr buffer = Marshal.AllocHGlobal(len);
Marshal.Copy(buf, 0, buffer, len);
rtn = Marshal.PtrToStructure(buffer, type);
Marshal.FreeHGlobal(buffer);
return rtn;
}
public static byte[] StructToBytes(object structObj)
{
//得到结构体的大小
int size = Marshal.SizeOf(structObj);
//创建byte数组
byte[] bytes = new byte[size];
//分配结构体大小的内存空间
IntPtr structPtr = Marshal.AllocHGlobal(size);
//将结构体拷到分配好的内存空间
Marshal.StructureToPtr(structObj, structPtr, false);
//从内存空间拷到byte数组
Marshal.Copy(structPtr, bytes, 0, size);
//释放内存空间
Marshal.FreeHGlobal(structPtr);
//返回byte数组
return bytes;
}The second is some simple memory operations. Due to the c# encapsulation, we cannot access memory data directly through pointers like c++. We need to use the Buffer class. The specific usage is as follows:
//用于同时操作内存块中的数据
public static void BlockCopy (Array src, int srcOffset, Array dst, int dstOffset, int count);It can copy the data interception count bits of the src starting position offset srcOffset to the inner thick area of the count length behind the dst starting position offset dstOffset.
If you want to intercept the data in the specified range of a byte[] type variable, you need to use the following function:
bytes.Skip(recvLenReal).Take(bytes.Length - recvLenReal).ToArray();Skip the length of recvLenReal from the beginning and then intercept bytes.Length - recvLenReal long data and convert it into a variable of type byte[].
3. Data frame encapsulation logic
The implementation of custom data packets is mainly divided into two parts: data header and data body.
The packet header is defined as follows:
| Data header | type of data | Data subject length | ||||
The two-byte data header is used to distinguish the protocol; the one-byte data type is used to identify the data type of the subsequent data body, such as: pictures or text or other content; the four-byte data body length Together they form an int type variable, which can indicate the total length of subsequent data.
The implementation method of this data header in unity and python is as shown in the figure below:
public struct DataHeader
{
public byte header1;
public byte header2;
public byte data_type;
public int data_len;
}The python part of the code is shown above and will not be repeated here.
4. Implementation of packet authentication for socket communication between python and unity
Regarding the data packet authentication code, both use the same logic. Only the corresponding logic processing annotations are provided here, but there are slight differences in the implementation of the code.
def process_raw_data(self, recv_data):
'''
关于操作：
本函数应该具有递归功能，否则无法处理复杂任务
关于消息接收的逻辑处理：
1. 首先判断当前是否已经接收过帧头 (self.frame_info.data_type is not None)
接收过：
根据帧头的数据长度接收对应的数据体内容
没接收过：
判断当前接收的数据长度是否满足帧头的长度
满足：尝试解析
解析失败：正常传输数据
解析成功：如果有其他的数据，继续接收处理后续的数据
不满足：将本次数据输出，丢弃此次的数据 !!!
'''
# 如果已经接收过数据头，直接继续接收内容
if self.frame_info.data_type is not None:
# 首先计算剩余的数据包长度
recv_len_left = self.frame_info.data_all_length - self.frame_info.data_recv_length
# 然后计算本次可以接收的数据长度，选取数据长度和剩余接收长度的最小值
recv_len_real = min(recv_len_left, len(recv_data))
self.frame_info.data_buffer += recv_data[:recv_len_real]
# 更新对应的接收的数据长度
self.frame_info.data_recv_length = len(self.frame_info.data_buffer)
# 判断当前是否已经接受完本帧的内容
if self.frame_info.data_recv_length >= self.frame_info.data_all_length:
# 根据回调函数返回对应的内容
if self.callback_fun is not None:
self.callback_fun(self.frame_info.data_buffer)
# 从剩余的数据中尝试检索出对应的数据头
# 首先更新 recv_data 的数据的内容
# print(self.frame_info.data_buffer)
self.frame_info.reset()
recv_data = recv_data[recv_len_real:len(recv_data)]
if len(recv_data) != 0:
self.process_raw_data(recv_data)
else:
return
# 从剩余的数据中尝试解析数据头
else:
if len(recv_data) >= self.header_length:
ret = self.process_protocol(recv_data[:self.header_length])
if ret[0]:
# 打印出协议对应的内容
# print(ret[1])
self.frame_info.set(ret[1][1], ret[1][2])
# 此处还得继续判断当前是否转换完了，如果没有的话需要继续转换接收到的内容
recv_data = recv_data[self.header_length:len(recv_data)]
if len(recv_data) != 0:
self.process_raw_data(recv_data)
else:
print(recv_data)
else:
print(recv_data) private void ProcessRawData(ref byte[] bytes)
{
// 关于操作：
// 本函数应该具有递归功能，否则无法处理复杂任务
// 关于消息接收的逻辑处理：
// 1.首先判断当前是否已经接收过帧头(self.frame_info.data_type is not None)
// 接收过：
// 根据帧头的数据长度接收对应的数据体内容
// 没接收过：
// 判断当前接收的数据长度是否满足帧头的长度
// 满足：尝试解析
// 解析失败：正常传输数据
// 解析成功：如果有其他的数据，继续接收处理后续的数据
// 不满足：将本次数据输出，丢弃此次的数据!!!
if (frameInfo.sPyDataTest.data_type != 0)
{
int recvLenLeft = frameInfo.sPyDataTest.data_len - frameInfo.data_recv_len;
int recvLenReal = Math.Min(recvLenLeft, bytes.Length);
frameInfo.AddBufferData(bytes.Skip(0).Take(recvLenReal).ToArray());
frameInfo.data_recv_len = frameInfo.GetBufferLength();
if (frameInfo.data_recv_len >= frameInfo.sPyDataTest.data_len)
{
//Debug.Log("接收成功！");
socketStatus = SocketStatus.eSocketReceived;
if (callBackFun != null)
{
callBackFun(socketStatus, frameInfo.data_buffer);
}
frameInfo.Reset();
bytes = bytes.Skip(recvLenReal).Take(bytes.Length - recvLenReal).ToArray();
if (bytes.Length != 0)
{
ProcessRawData(ref bytes);
}
}
else
{
return;
}
}
else
{
if (recvBuffer != null)
{
byte[] newBytes = new byte[recvBuffer.Length + bytes.Length];
Buffer.BlockCopy(recvBuffer, 0, newBytes, 0, recvBuffer.Length);
Buffer.BlockCopy(bytes, 0, newBytes, recvBuffer.Length, bytes.Length);
bytes = newBytes;
recvBuffer = null;
}
if (bytes.Length >= frameInfo.StructLength)
{
bool ret = true;
ret = frameInfo.IsHeaderMatch(bytes.Take(frameInfo.StructLength).ToArray());
if (ret == true)
{
//Debug.Log(frameInfo.sPyDataTest);
frameInfo.Set(frameInfo.sPyDataTest.data_type, frameInfo.sPyDataTest.data_len);
bytes = bytes.Skip(frameInfo.StructLength).Take(bytes.Length - frameInfo.StructLength).ToArray();
if (bytes.Length != 0)
{
ProcessRawData(ref bytes);
}
}
else
{
Debug.Log(bytes);
}
}
else
{
recvBuffer = bytes;
Debug.Log(bytes);
}
}
}Summarize
Here we only provide code implementation methods and principle explanations for the corresponding key steps. If there are any errors, please correct me!