socket & sticky learning package python

7.4 socket

[Important]

Avoid learning each layer interface, and protocol usage, socket has a good package of all the interfaces, use these interfaces or methods can be convenient and efficient, improve development efficiency.

socket in python is a module by using the function provided by the learning module, and establishing a communication connection between two processes (ip + port).

flow chart

Server initialized socket, then bind bind port, port listen listen, call accept rammed live program, connection requests from clients; client initialization socket, connect the server, the connection is successful, the client sends data to the server. after receiving the data returned by the server, the client reads the data request to close, once the end of the interaction.

socket module

Cyclic communication

# 服务器端
import socket
server = socket.socket()  # 创建server服务端,可以不写，默认是socket.AF_INET,socket.SOCK_STREAM
server.bind(('127.0.0.1',8003))  # 绑定IP地址和端口
server.listen(5)  # 设置最大连接数
print('listening')
conn,addr = server.accept() # 进入监听状态

while 1:
    from_client_data = conn.recv(1024).decode('utf-8')  # 设置最大字节数
    if from_client_data.upper() == 'Q':   #判断对方是否要求关闭连接
        break
    else:
        print(f"来自{addr}的消息:\033[1;32m{from_client_data}\033[0m")
        se = input('>>>').encode('utf-8')
        conn.send(se)

conn.close()  # 关闭连接
server.close()  # 关闭服务端

# 客户端
import socket

client = socket.socket()   # 可以不写，默认是socket.AF_INET,socket.SOCK_STREAM

client.connect(('127.0.0.1',8003)) # 与服务器建立连接,ip地址与端口号必须要与服务器端一致

while 1:
    se = input('>>>')
    if se.upper() == 'Q':
        client.send('q'.encode('utf-8'))
        break
    client.send(se.encode('utf-8'))

    from_server_data = client.recv(1024) # 设置最大字节数
    print(f"来自服务器的消息：\033[1;32m {from_server_data.decode('utf-8')}\033[0m")

client.close()

Server Edition has been listening [important]

# 服务器端
import socket

server = socket.socket()
server.bind(('127.0.0.1',8003))
server.listen(5)
print('listening')

while 1:
    conn,addr = server.accept()
    while 1:
        try :
            from_client_data = conn.recv(1024).decode('utf-8')
            if from_client_data.upper() == 'Q':
                break
            else:
                print(f"来自{addr}的消息:\033[1;32m{from_client_data}\033[0m")
                se = input('>>>').encode('utf-8')
                conn.send(se)
        except ConnectionResetError:
            break
    conn.close()
server.close()

# 客户端
import socket

client = socket.socket()
client.connect(('127.0.0.1',8003))

while 1:
    se = input('>>>')
    if se.upper() == 'Q':
        client.send('q'.encode('utf-8'))
        break
    client.send(se.encode('utf-8'))

    from_server_data = client.recv(1024).decode('utf-8')
    print(f"来自服务器的消息：\033[1;32m{from_server_data}\033[0m")

client.close()

# 这里要注意多client同时访问server时，会按照顺序，断开一个连接后继续下一个连接

Remote command execution

subprocess application, create a process

import subprocess

obj = subprocess.Popen('dir',
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         )

print(obj.stdout.read().decode('gbk'))  # 正确命令
print('error:',obj.stderr.read().decode('gbk'))  # 错误命令

Remote Use

import socket
import subprocess
server = socket.socket()
server.bind(('127.0.0.1',8004))
server.listen(5)
print('listening')

while 1:
    conn,addr = server.accept()
    while 1:
        rec = conn.recv(1024).decode('utf-8')
        if rec.upper() == 'Q':
            break
        else:
            obj = subprocess.Popen(rec,
                                   shell=True,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   )
            se = obj.stdout.read().decode('gbk')+obj.stderr.read().decode('gbk')
            conn.send(se.encode('utf-8'))
    conn.close()
server.closed()

import socket

client = socket.socket()
client.connect(('127.0.0.1',8004))

while 1:
    se = input('请输入命令')
    if se.upper() == 'Q':
        client.send('q'.encode('utf-8'))
        break
    else:
        client.send(se.encode('utf-8'))

    re = client.recv(1024).decode('utf-8')
    print(re)

client.close()
# 如果返回的数据过多，会有黏包现象

7.5 sticky package

cause

Buffer

Without the buffer, send and receive data will be affected by fluctuations in the network, affecting upload and download data

Although the buffer to solve the efficiency of transmission of uploading and downloading, but the sticky bag issue

Produce condition

1.recv will produce sticky package. If the received recv data <data buffer, a data buffer receiving data is incomplete and behind the recv stick together, streaming data

2. Continuous send small amounts of data, first buffer. Since the Nagle algorithm, the data sender sends if relatively small, may be temporarily stored in a buffer, coupled with the data transfer to the application layer of the TCP fast, then the two will stick together application layer packets, and finally send TCP a TCP packet to a receiving end

solution

struct module

According to the specified format to convert into a fixed-length type python bytes byte stream

format	C language type	Python type	Gauge
x	Padding bytes	No value
c	char	string of length 1	1
b	signed	integer	1
B	unsigned char	integer	1
?	_Bool	bool	1
h	short	integer	2
H	unsigned short	integer	2
i	int	integer	4
I	unsigned int	integer	4
l	long	integer	4
L	unsigned long	long	4
q	long long	long	8
Q	unsigned long long	long	8
n	ssize_t	intter
N	size_t	intter
f	float	float	4
d	double	float	8
s	char[]	string
p	char[]	string
P	void *	long

import struct
# 将一个数字转化成等长度的bytes类型。
ret = struct.pack('q', 154365400024634546545646546)
print(ret, type(ret), len(ret))

# 通过unpack反解回来
ret1 = struct.unpack('i',ret)[0]
print(ret1)

方案一：

​ 1.在第二次向对方发送数据之前，先把缓冲区的数据全部取出
​ 2.如何限制循环次数?当接收数据的总字节数数等于发送数据的总字节数时，停止循环；
​ 3.服务端先send要发送数据的总字节数；
​ 4.发送端制作报头，固定头部长度，使用struct模块，pack转换成等长度4个字节bytes类型，发送到接收端，接收端再使用unpack将字节数转换回来；

代码实现

服务器端

import socket
import subprocess
import struct

server = socket.socket()
server.bind(('127.0.0.1',8004))
server.listen(5)
print('listening')

# 接收连接
while 1:
    conn,addr = server.accept()
    while 1:
        rec = conn.recv(1024).decode('utf-8')
        obj = subprocess.Popen(rec,
                               shell=True,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               )
        se = obj.stdout.read().decode('gbk')+obj.stderr.read().decode('gbk')
        # 制作报头
        total_size = len(se)
        # 将长度不固定的int类型报头，转成固定长度bytes4个字节
        # 将一个数字转换成等长度的bytes类型
        total_size_bytes = struct.pack('i',total_size)
        # 发送报头
        conn.send(total_size_bytes)
        # 发送原始数据
        conn.send(se.encode('gbk'))
    conn.close()
server.closed()

客户端

import struct
import socket

client = socket.socket()
client.connect(('127.0.0.1',8004))
# 发送消息
while 1:

    se = input('请输入命令')
    client.send(se.encode('utf-8'))
    # 1-接收报头
    head_bytes = client.recv(4)
    # 2-将报头反解回int类型
    total_size = struct.unpack('i',head_bytes)[0]   #unpack返回的一个元组，里边只有一个反解的元素
    # 3-循环接收数据
    total_data = b''  #设定一个初始值
    while len(total_data.decode('gbk')) < total_size:
        total_data += client.recv(1024)

    print(total_data.decode('gbk'))

client.close()

存在的问题：

​ 数据量较大的数据，使用struct时会报错；

​ 报头信息不可能只含有数据的大小；

方案二：

为了解决以上问题，我们引入方案二：

自定义一种传输报文：

• 将传送数据的信息汇集成字典，字典中记录了传输数据的MD5值，filename以及filesize,使用json模块把字典转换成bytes类型，将其加到传输数据流的字典头；
• 记录字典bytes的长度，使用struct模块转成4字节的固定长度，然后把它作为长度头加到数据流的最开始；
• 将 [长度头+字典头+传输数据] 一起发送给对方
• 对方接到数据流之后，首先取4个字节，求出字典头的长度；取字典头长度的数据，进行格式转换，得到字典；
• 循环接收数据，将得到的数据流与字典中的MD5进行校验，校验通过，转码显示数据，校验不通过，则不显示；

服务器端

while 1:
    conn, addr = server.accept()

    while 1:
        try :
            re = conn.recv(1024).decode('gbk')
            obj = subprocess.Popen(re,
                                   shell=True,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   )

            result = obj.stdout.read() + obj.stderr.read()  # 获得的是gbk转码后的数据

            size = len(result)  #求原数据的大小

            # 做一下文件校验的MD5序列
            ret = hashlib.md5()
            ret.update(result)
            md5 = ret.hexdigest()
            # 制作报头
            head_dict = {
                'md5':md5,
                'filename':re,
                'filesize':size
            }

            # 将报头字典转换成json序列
            head_dict_json = json.dumps(head_dict)

            # 将json字符串转换成bytes
            head_dict_json_bytes = head_dict_json.encode('gbk')

            # 获取报头的长度
            head_len = len(head_dict_json_bytes)

            # 将head_len转成固定长度
            head_len_bytes = struct.pack('i',head_len)

            # 发送固定的4个字节
            conn.send(head_len_bytes)
            # 发送字典报头
            conn.send(head_dict_json_bytes)
            # 发送文件
            conn.send(result)

        except Exception:
            break
    conn.close()
server.close()

客户端

import socket
import struct
import json
import hashlib

client = socket.socket()
client.connect(('127.0.0.1',8005))

while 1:
    se = input('>>>').strip().encode('gbk')
    client.send(se)

    # 接受4个字节，获得头部的字典长度
    head_bytes = client.recv(4)
    # 将字典长度转成int类型
    head_dic_len = struct.unpack('i',head_bytes)[0]

    # 接收字典的数据流
    head_dic_bytes = client.recv(head_dic_len).decode('gbk')
    # 转换成字典模式
    head_dict = json.loads(head_dic_bytes)

    # 定义一个接收主内容的句柄
    file = b''
    while len(file) < head_dict['filesize']:
        file += client.recv(1024)

    # 计算一下file的MD5
    ret = hashlib.md5()
    ret.update(file)
    md_5 = ret.hexdigest()

    if md_5 == head_dict['md5']:
        print('文件校验成功')
        print(file.decode('gbk'))
    else:
        print('文件校验未通过')
        break
client.close()