tcp protocol sticky packet problem and socketserver basics

1. Implement remote command execution based on tcp protocol

1.1 Server

The server should meet two characteristics:
1) Always provide services externally
2) Serve multiple clients concurrently

import subprocess
from socket import *

server=socket(AF_INET,SOCK_STREAM)
server.setsockopt(SOL_SOCKET,SO_REUSEADDR,1) #就是它，在bind前加
server.bind(('127.0.0.1',8082))
server.listen(5)

#  服务端应该做两件事

# 第一件事：循环地从板连接池中取出链接请求与其建立双向链接，拿到链接对象

while True:
    conn,client_addr=server.accept()

# 第二件事：拿到链接对象，与其进行通信循环

while True:
    try:
        cmd=conn.recv(1024)
        if len(cmd) == 0:break
        obj=subprocess.Popen(cmd.decode('utf-8'),
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE
                         )

​        stdout_res=obj.stdout.read()
​        stderr_res=obj.stderr.read()
​        print(len(stdout_res)+len(stderr_res))

        # conn.send(stdout_res+stderr_res) # ???

​        conn.send(stdout_res)
​        conn.send(stderr_res)

        # with open("1.mp4",mode='rb') as f:

        #     for line in f:

        #         conn.send(line)

​    except Exception:
​        break
conn.close()

1.2 Client

from socket import *

client=socket(AF_INET,SOCK_STREAM)
client.connect(('127.0.0.1',8082))

while True:
    cmd=input('请输入命令>>：').strip()
    if len(cmd) == 0:continue
    client.send(cmd.encode('utf-8'))

# 解决粘包问题思路：

# 1、拿到数据的总大小total_size

# 2、recv_size=0,循环接收，每接收一次，recv_size+=接收的长度

# 3、直到recv_size=total_size，结束循环

cmd_res=client.recv(1024) # 本次接收，最大接收1024Bytes
print(cmd_res.decode('utf-8')) # 强调：windows系统用gbk

The reason for the sticky packet problem
1) tcp is a streaming protocol, the data is glued together like a water stream, there is no boundary distinction
2) The collected data is confiscated clean, there are residues, and the next result will be confused

The core method of solution is: clean every time, do not leave any residue

Second, the udp protocol has no sticky packets

2.1 udp server

import socket

server=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
server.bind(('127.0.0.1',8080))
res1=server.recvfrom(2) # b"hello"
print(res1)
res2=server.recvfrom(3) # b"world"
print(res2)

server.close()

2.2 udp client

import socket

client=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
client.sendto(b'hello',('127.0.0.1',8080))
client.sendto(b'world',('127.0.0.1',8080))

client.close()

3. Solve the sticky package problem

3.1 Server

The server should meet two characteristics:
1) Always provide services externally
2) Serve multiple clients concurrently

import subprocess
import struct
from socket import *

server=socket(AF_INET,SOCK_STREAM)
server.setsockopt(SOL_SOCKET,SO_REUSEADDR,1) #就是它，在bind前加
server.bind(('127.0.0.1',8083))
server.listen(5)

#  服务端应该做两件事

# 第一件事：循环地从板连接池中取出链接请求与其建立双向链接，拿到链接对象

while True:
    conn,client_addr=server.accept()

# 第二件事：拿到链接对象，与其进行通信循环

while True:
    try:
        cmd=conn.recv(1024)
        if len(cmd) == 0:break
        obj=subprocess.Popen(cmd.decode('utf-8'),
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE
                         )

​        stdout_res=obj.stdout.read()
​        stderr_res=obj.stderr.read()
​        total_size=len(stdout_res)+len(stderr_res)

        # 1、先发头信息（固定长度的bytes）：对数据描述信息

        # int->固定长度的bytes

​        header=struct.pack('i',total_size)
​        conn.send(header)

        # 2、再发真实的数据

​        conn.send(stdout_res)
​        conn.send(stderr_res)

​    except Exception:
​        break
conn.close()

3.2 Client

import struct
from socket import *

client=socket(AF_INET,SOCK_STREAM)
client.connect(('127.0.0.1',8083))

while True:
    cmd=input('请输入命令>>：').strip()
    if len(cmd) == 0:continue
    client.send(cmd.encode('utf-8'))

# 解决粘包问题思路：

# 一、先收固定长度的头：解析出数据的描述信息，包括数据的总大小total_size

header=client.recv(4)
total_size=struct.unpack('i',header)[0]

# 二、根据解析出的描述信息，接收真实的数据

# 2、recv_size=0,循环接收，每接收一次，recv_size+=接收的长度

# 3、直到recv_size=total_size，结束循环

recv_size = 0
while recv_size < total_size:
    recv_data=client.recv(1024)
    recv_size+=len(recv_data)
    print(recv_data.decode('utf-8'),end='')
else:
    print()

The reason for the sticky packet problem
1) tcp is a streaming protocol, the data is glued together like a water stream, there is no boundary distinction
2) The collected data is confiscated clean, there are residues, and the next result will be confused

The core method of solution is: clean every time, do not leave any residue

Fourth, solve the sticky package problem (ultimate version)

4.1 Server

The server should meet two characteristics:
1) Always provide services externally
2) Serve multiple clients concurrently

import subprocess
import struct
import json
from socket import *

server=socket(AF_INET,SOCK_STREAM)
server.setsockopt(SOL_SOCKET,SO_REUSEADDR,1) #就是它，在bind前加
server.bind(('127.0.0.1',8083))
server.listen(5)

#  服务端应该做两件事

# 第一件事：循环地从板连接池中取出链接请求与其建立双向链接，拿到链接对象

while True:
    conn,client_addr=server.accept()

# 第二件事：拿到链接对象，与其进行通信循环

while True:
    try:
        cmd=conn.recv(1024)
        if len(cmd) == 0:break
        obj=subprocess.Popen(cmd.decode('utf-8'),
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE
                         )

​        stdout_res=obj.stdout.read()
​        stderr_res=obj.stderr.read()
​        total_size=len(stdout_res)+len(stderr_res)

        # 1、制作头

​        header_dic={
​            "filename":"a.txt",
​            "total_size":total_size,
​            "md5":"123123xi12ix12"
​        }

​        json_str = json.dumps(header_dic)
​        json_str_bytes = json_str.encode('utf-8')

        # 2、先把头的长度发过去

​        x=struct.pack('i',len(json_str_bytes))
​        conn.send(x)

        # 3、发头信息

​        conn.send(json_str_bytes)

        # 4、再发真实的数据

​        conn.send(stdout_res)
​        conn.send(stderr_res)

​    except Exception:
​        break
conn.close()

4.2 Client

import struct
import json
from socket import *

client=socket(AF_INET,SOCK_STREAM)
client.connect(('127.0.0.1',8083))

while True:
    cmd=input('请输入命令>>：').strip()
    if len(cmd) == 0:continue
    client.send(cmd.encode('utf-8'))

# 接收端

# 1、先手4个字节，从中提取接下来要收的头的长度

x=client.recv(4)
header_len=struct.unpack('i',x)[0]

# 2、接收头，并解析

json_str_bytes=client.recv(header_len)
json_str=json_str_bytes.decode('utf-8')
header_dic=json.loads(json_str)
print(header_dic)
total_size=header_dic["total_size"]

# 3、接收真实的数据

recv_size = 0
while recv_size < total_size:
    recv_data=client.recv(1024)
    recv_size+=len(recv_data)
    print(recv_data.decode('utf-8'),end='')
else:
    print()

The reason for the sticky packet problem
1) tcp is a streaming protocol, the data is glued together like a water stream, there is no boundary distinction
2) The collected data is confiscated clean, there are residues, and the next result will be confused

The core method of solution is: clean every time, do not leave any residue

Five, the basic use of the socketserver module

5.1 Use based on tcp protocol

5.1.1 Server

import socketserver

class MyRequestHandle(socketserver.BaseRequestHandler):
    def handle(self):
        # 如果tcp协议，self.request=>conn
        print(self.client_address)
        while True:
            try:
                msg = self.request.recv(1024)
                if len(msg) == 0: break
                self.request.send(msg.upper())
            except Exception:
                break
        self.request.close()



#  服务端应该做两件事

# 第一件事：循环地从半连接池中取出链接请求与其建立双向链接，拿到链接对象

s=socketserver.ThreadingTCPServer(('127.0.0.1',8889),MyRequestHandle)
s.serve_forever()

# 等同于

# while True:

#     conn,client_addr=server.accept()

#     启动一个线程(conn,client_addr)

# 第二件事：拿到链接对象，与其进行通信循环===>handle

5.1.2 Client

from socket import *

client=socket(AF_INET,SOCK_STREAM)
client.connect(('127.0.0.1',8889))

while True:
    msg=input('请输入命令>>：').strip()
    if len(msg) == 0:continue
    client.send(msg.encode('utf-8'))

res=client.recv(1024)
print(res.decode('utf-8'))

5.2 Use based on udp protocol

5.2.1 Server

import socketserver

class MyRequestHanlde(socketserver.BaseRequestHandler):
    def handle(self):
        client_data=self.request[0]
        server=self.request[1]
        client_address=self.client_address
        print('客户端发来的数据%s' %client_data)
        server.sendto(client_data.upper(),client_address)


s=socketserver.ThreadingUDPServer(("127.0.0.1",8888),MyRequestHanlde)
s.serve_forever()

# 相当于:只负责循环地收

# while True:

#     data,client_addr=server.recvfrom(1024)

#     启动一个线程处理后续的事情(data,client_addr)

5.2.2 Client

import socket

client=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # 流式协议=》tcp协议

while True:
    msg=input('>>>: ').strip()
    client.sendto(msg.encode('utf-8'),('115.29.65.16',8888))
    res=client.recvfrom(1024)
    print(res)

client.close()