[Linux development—I/O reuse]

1. Background

Disadvantages of multi-process servers:

- 1，需要大量的运算
- 2，需要大量的内存空间

Application of reuse technology on the server side:

2. I/O reuse

• The meaning of reuse:
  • 1. Improve the efficiency of physical equipment
  • 2. Use the fewest physical elements to deliver the most data

For more comparisons between select and epoll, I personally read some other people’s posts. You can refer to the comparison between select and epoll.

1.Select model

0. Advantages and Disadvantages:

• Advantages: A single thread can serve multiple clients.
• shortcoming:
  • 1. The number of service clients is limited and cannot exceed 1024.
  • 2. Accept needs to be polled and monitored. If the number of connections is relatively large, it will consume too much performance.

1. Understanding the select model:

1. fd_set sets the file descriptor:

• The select function monitors multiple (no more than 1024) file descriptors
  
• fd_set structure
• FD_ZERO (fd set *fdset): Initialize all bits of the fd set variable to 0.
• FD_SET (int fd, fd set*fdset): Register the information of the file descriptor fd in the variable pointed to by the parameter dset.
• FD_CLR (int fd, fd set*fdset): Clear the information of the file descriptor fd from the variable pointed to by the parameter fdset.
• FD_ISSET (int fd, fd_set*fdset): If the variable pointed to by the parameter fdset contains information about the file descriptor fd, it returns "true".

2. Call the select function:

#include<sys/select.h>
#include <sys/time.h>
int select(int nfds, fd_set*readset, fd_set* writeset, fd_set*exceptset, const struct timeval * timeout);	
// →成功时返回大于0的值，失败时返回-1。

• nfds : The number of monitoring object file descriptors.
• readset : used to check readability,
• writeset : used to check writability,
• exceptset : used to check out-of-band data,
• timeout : A pointer to a timeval structure, used to determine the maximum time select waits for I/O. If it is empty, it will wait forever .

2. Select model instance:

select focuses on the server side

void select_cs_connect(char* arg)
{
    
    
    if (strcmp(arg, "s") == 0)//如果输入s，走服务端路线
    {
    
    
        select_server();
    }
    else
    {
    
    
        fputs("Please input your name:", stdout);
        scanf("%s", name);
        select_client();
    }
}

1. Server

//select函数
#include<sys/select.h>
#include <sys/time.h>

void select_server()
{
    
    

    int serv_sock;
    struct sockaddr_in serv_adr;
    serv_sock = socket(PF_INET, SOCK_STREAM, 0);
    if (serv_sock == -1)
    {
    
    
        printf("create socket error:%d %s\n", errno, strerror(errno));
        return;
    }

    memset(&serv_adr, 0, sizeof(serv_adr));
    serv_adr.sin_family = AF_INET;
    serv_adr.sin_addr.s_addr = htonl(INADDR_ANY);
    serv_adr.sin_port = htons(9527);

    if (bind(serv_sock, (struct sockaddr*)&serv_adr, sizeof(serv_adr)) == -1)
    {
    
    
        error_handling("thread server bind error");
        printf("bind error msg:%d %s\n", errno, strerror(errno));
        return;
    }

    if (listen(serv_sock, 5) == -1)
    {
    
    
        error_handling("thread server listen error");
        printf("listen error msg:%d %s\n", errno, strerror(errno));
        return;
    }

    printf("create server socket success!\n");

    int clnt_sock;
    struct sockaddr_in clnt_adr;
    socklen_t clnt_adr_sz = sizeof(clnt_adr);
    
    fd_set reads, copy_reads;
    FD_ZERO(&reads);
    FD_SET(serv_sock, &reads);
    timeval timeout = {
    
    5, 5000};//设置5.5秒超时，结构体中，第二个参数为微秒结构0.5秒 = 5000000
    int fd_max = serv_sock;
    
    while (1)
    {
    
    
        copy_reads = reads;
        int fd_num = select(fd_max + 1, &copy_reads, NULL, NULL, &timeout);//套接字最大再加1，范围全部覆盖
        if (fd_num == -1)
        {
    
    
            printf("select error msg:%d %s\n", errno, strerror(errno));

            close(serv_sock);
            return;
        }
        if (fd_num == 0) continue;

        printf("fd_num is:%d\n",fd_num);
		//轮询
        for (int i = 0; i < fd_max + 1; i++)
        {
    
    
            if (FD_ISSET(i, &copy_reads))
            {
    
    
                if (i == serv_sock)     // connection request!
                {
    
    
                    clnt_sock =
                        accept(serv_sock, (struct sockaddr*)&clnt_adr, &clnt_adr_sz);
                    FD_SET(clnt_sock, &reads);
                    if (fd_max < clnt_sock)
                        fd_max = clnt_sock;
                    printf("connected client: %d \n", clnt_sock);
                }
                else    // read message!
                {
    
    
                    char buf[256] = "";
                    ssize_t str_len = read(i, buf, BUF_SIZE);
                    if (str_len == 0)    // close request!
                    {
    
    
                        FD_CLR(i, &reads);
                        close(i);
                        printf("closed client: %d \n", i);
                    }
                    else
                    {
    
    
                        write(i, buf, str_len);    // echo!
                    }
                }
            }
        }
    }

    close(serv_sock);
}

2. Client

// 客户端 发送消息
char name[64] = "[MOON]";
void* client_send_msg(void* arg)
{
    
    
    pthread_detach(pthread_self());
    int clnt_sock = *(int*)arg;//取出当前线程的socket
    char msg[256] = "";
    char buffer[1024];
    while (1)
    {
    
    
        memset(buffer, 0, sizeof(buffer));
        fgets(msg, sizeof(msg), stdin);//对文件的标准输入流操作 读取buffer的256字节
        if (strcmp(msg, "q\n") == 0 || (strcmp(msg, "Q\n") == 0)) {
    
    
            break;
        }

        
        if (strcmp(msg, "") == 0)
        {
    
    
            continue;
        }

        snprintf(buffer, sizeof(buffer), "%s: %s", name, msg);
        size_t len = strlen(buffer);
        size_t send_len = 0;

        //当数据量很大时，并不能一次把所有数据全部发送完，因此需要分包发送
        while (send_len < len)
        {
    
    
            ssize_t ret = write(clnt_sock, buffer + send_len, len - send_len);//send_len 记录分包的标记
            if (ret <= 0) {
    
    //连接出了问题
                fputs("may be connect newwork failed,make client write failed!\n", stdout);
                break;
            }
            send_len += (size_t)ret;
        }       
    };

    sem_post(&semid);
    pthread_exit(NULL);
}
//客户端-接收消息
void* client_recv_msg(void* arg)
{
    
    
    pthread_detach(pthread_self());
    int clnt_sock = *(int*)arg;//取出当前线程的socket
    char buffer[1024] = "";
    while (1)
    {
    
    
        size_t ret = read(clnt_sock, buffer, sizeof(buffer));
        if (ret <= 0) {
    
    //连接出了问题
            fputs("client read failed!\n", stdout);
            break;
        }
        fputs(buffer, stdout);
        memset(buffer, 0, ret);//处理完消息及时重置内存
    };
    sem_post(&semid);
    pthread_exit(NULL);
}

void select_client()
{
    
    
    struct sockaddr_in clnt_adr;

    socklen_t clnt_adr_sz = sizeof(clnt_adr);

    int clnt_sock = socket(PF_INET, SOCK_STREAM, 0);

    memset(&clnt_adr, 0, clnt_adr_sz);
    clnt_adr.sin_family = AF_INET;
    clnt_adr.sin_addr.s_addr = inet_addr("127.0.0.1");
    clnt_adr.sin_port = htons(9527);
    if (connect(clnt_sock, (struct sockaddr*)&clnt_adr, clnt_adr_sz) == -1)
    {
    
    
        printf("connect error msg:%d %s\n", errno, strerror(errno));
        return;
    }

    pthread_t thread_send, thread_recv;

    sem_init(&semid, 0, -1);
    pthread_create(&thread_send, NULL, client_send_msg, &clnt_sock);//消息clnt_sock为局部变量
    pthread_create(&thread_recv, NULL, client_recv_msg, &clnt_sock);//消息clnt_sock为局部变量


    sem_wait(&semid);

    close(clnt_sock);
}

2. Epoll model

• Solve the shortcomings of the select model (select polling device consumes performance and the problem of limited number of connected devices)
• Disadvantages of the Select model:
  • The common loop statement for all file descriptors after calling the select function . If there are many inactive users, the select efficiency is relatively low. If there are many active users, the select efficiency is higher than epoll.
    Each time you call the select function, you need to pass the monitoring object information to the function .
  • After calling the select function, the changed file descriptors are not gathered together individually, but the changed file descriptors are found by observing the changes in the fd_set variable as the monitoring object, so the loop for all monitoring objects cannot be avoided. statement. Moreover, the fd_set variable as the monitoring object will change, so the original information should be copied and saved before calling the select function, and new monitoring object information should be passed every time the select function is called.
• solution:
  • "Transfer the monitoring object to the operating system once. If the monitoring scope or content changes, only the changed matters will be notified." In
    this way, there is no need to pass the monitoring object information to the operating system every time the select function is called, but the premise is that the operating system supports this How it is handled (how much and how each operating system supports it differs). The support method for Linux is epoll, and the support method for Windows is IOCP . epoll这个是Linux独有的函数.

1. Understanding the Epoll model

Epoll's three major functions: epoll_create, epoll_wait, epoll_ctl

1，epoll_create

#include<sys/epoll.h>
int epoll_create(int size);
//→成功时返回epoll文件描述符，失败时返回-1。

• size : The size of the epoll instance.
  This function has been added since version 2.3.2, and version 2.6 has introduced the kernel. The
  latest stable kernel version of Linux has reached 5.8.14, and the long-term support version has reached 5.4.70.
  Linux starting from the 2.6.8 kernel will ignore this parameter. But it must be greater than 0.
  This is a unique function of Linux.

2，epoll_ctl

#include<sys/epoll.h>
int epoll_ctl(int epfd, int op, int fd, struct epoll_event* event);
//→成功时返回0，失败时返回-1。

• The file descriptor used by epfd to register the epoll routine of the monitored object.
• op is used to specify operations such as adding, deleting or changing monitoring objects.
  • EPOLL_CTL_ADD increase
  • EPOLL_CTL_MOD modification
  • EPOLL_CTL_DEL delete
• fd needs to register the monitoring object file descriptor.
• The event type of the event monitoring object.
  • EPOLLIN : When data needs to be read.
  • EPOLLOUT : The output buffer is empty and data can be sent immediately.
  • EPOLLPRI : When OOB data is received.
  • EPOLLRDHUP : Disconnected or half-closed situation, which is very useful in edge trigger mode.
  • EPOLLERR : When an error occurs.
  • EPOLLET : Get event notification in an edge-triggered manner.
  • EPOLLONESHOT : After an event occurs, the corresponding file descriptor no longer receives event notifications. Therefore, you need to pass EPOLLCTL_MOD to the second parameter of the epoll_ctl function and set the event again.

3，epoll_wait：

#include <sys/epoll.h>
int epoll_wait(int epfd, struct epoll_event*events,int maxevents,int timeout);
//→成功时返回发生事件的文件描述符数，失败时返回-1。

• epfd represents the file descriptor of the epol routine in the event monitoring range.
• events stores the structure address value of the file descriptor collection where the event occurred.
• The maximum number of events that can be saved in the second parameter of maxevents .
• Timeout : The waiting time in units of 1/1000 seconds. When -1 is passed, wait until the event occurs.

2. Epoll model example

1. Server

// epoll模型
#include <sys/epoll.h>
void epoll_server()
{
    
    
    int serv_sock;
    struct sockaddr_in serv_adr;
    serv_sock = socket(PF_INET, SOCK_STREAM, 0);
    if (serv_sock == -1)
    {
    
    
        printf("create socket error:%d %s\n", errno, strerror(errno));
        close(serv_sock);
        return;
    }

    memset(&serv_adr, 0, sizeof(serv_adr));
    serv_adr.sin_family = AF_INET;
    serv_adr.sin_addr.s_addr = htonl(INADDR_ANY);
    serv_adr.sin_port = htons(9527);

    if (bind(serv_sock, (struct sockaddr*)&serv_adr, sizeof(serv_adr)) == -1)
    {
    
    
        error_handling("thread server bind error");
        printf("bind error msg:%d %s\n", errno, strerror(errno));
        close(serv_sock);
        return;
    }

    if (listen(serv_sock, 5) == -1)
    {
    
    
        error_handling("thread server listen error");
        printf("listen error msg:%d %s\n", errno, strerror(errno));
        close(serv_sock);
        return;
    }

    printf("create server socket success!\n");

    int clnt_sock;
    struct sockaddr_in clnt_adr;
    socklen_t clnt_adr_sz = sizeof(clnt_adr);

    epoll_event event;
    int event_cnt;
    int epfd = epoll_create(1);
    if (epfd == -1)
    {
    
    
        printf("epoll_create error msg:%d %s\n", errno, strerror(errno));
        close(serv_sock);
        return;
    }

    epoll_event* all_events = new epoll_event[100];

    event.events = EPOLLIN;
    event.data.fd = serv_sock;
    epoll_ctl(epfd, EPOLL_CTL_ADD, serv_sock, &event);
    while (true)
    {
    
    
        event_cnt = epoll_wait(epfd, all_events, 100, 1000);//1000：等待1秒
        if (event_cnt == -1)
            printf("epoll_wait error msg:%d %s\n", errno, strerror(errno));
            break;

        if (event_cnt == 0)continue;
        for (int i = 0; i < event_cnt; i++)
        {
    
    
            int event_data_fd = all_events[i].data.fd;
            if (event_data_fd == serv_sock) {
    
    
                clnt_sock = accept(serv_sock, (sockaddr*)&clnt_adr, &clnt_adr_sz);
                event.events = EPOLLIN;
                event.data.fd = clnt_sock;
                epoll_ctl(epfd, EPOLL_CTL_ADD, clnt_sock, &event);
                printf("client is connect success! %d\n", clnt_sock);
            }
            else
            {
    
    
                char buf[256] = "";
                ssize_t str_len = read(event_data_fd, buf, sizeof(buf));
                if (str_len <= 0)    // close request!
                {
    
    
                    epoll_ctl(epfd, EPOLL_CTL_DEL, event_data_fd, NULL);
                    close(event_data_fd);
                    
                    printf("closed client: %d \n", i);
                }
                else
                {
    
    
                    write(event_data_fd, buf, str_len);    // echo!
                }
            }
        }

    }
    delete[] all_events;
    close(serv_sock);
    close(epfd);
}

void epoll_cs_connect(char* arg)
{
    
    
    if (strcmp(arg, "s") == 0)//如果输入s，走服务端路线
    {
    
    
        epoll_server();
    }
    else
    {
    
    
        fputs("Please input your name:", stdout);
        scanf("%s", name);
        select_client();
    }
}

2. Client

• The client code logic is the same as the client code in the select model instance.

3. Edge triggering and conditional triggering

1. Definition

• Conditional triggering (level-triggered, also known as level triggering) LT :
  As long as the condition is met, an event is triggered (as long as there is data that has not been obtained, the kernel will continue to notify you)

• Edge-triggered (edge-triggered) ET :
  Whenever the state changes, an event is triggered.
  "Take an example of reading a socket. Assume that after a long period of silence, 100 bytes are now coming. At this time, regardless of edge triggering or conditional triggering, A notification will be generated that the application is readable. The application reads 50 bytes, and then calls the api again to wait for the io event. At
  this time, the horizontally triggered api will immediately return a read ready notification to the user because there are still 50 bytes to read. .The
  edge-triggered API will fall into a long wait because the readable status has not changed . Therefore, when using the edge-triggered API, be careful to read the socket every time and return EWOULDBLOCK, otherwise the socket will be considered useless. And When using a conditionally triggered API, if the application does not need to write, do not pay attention to the socket writable event, otherwise a write ready notification will be returned immediately and infinitely.

• Select is a typical conditional trigger .

• If events are set to EPOLLET in epoll, it will be edge triggered.

2. epoll edge trigger case:

边缘触发：
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>

#define BUF_SIZE 4
#define EPOLL_SIZE 50
void setnonblockingmode(int fd);
void error_handling(char *buf);

int main(int argc, char *argv[])
{
    
    
	int serv_sock, clnt_sock;
	struct sockaddr_in serv_adr, clnt_adr;
	socklen_t adr_sz;
	int str_len, i;
	char buf[BUF_SIZE];

	struct epoll_event *ep_events;
	struct epoll_event event;
	int epfd, event_cnt;

	if(argc!=2) {
    
    
		printf("Usage : %s <port>\n", argv[0]);
		exit(1);
	}

	serv_sock=socket(PF_INET, SOCK_STREAM, 0);
	memset(&serv_adr, 0, sizeof(serv_adr));
	serv_adr.sin_family=AF_INET;
	serv_adr.sin_addr.s_addr=htonl(INADDR_ANY);
	serv_adr.sin_port=htons(atoi(argv[1]));
	
	if(bind(serv_sock, (struct sockaddr*) &serv_adr, sizeof(serv_adr))==-1)
		error_handling("bind() error");
	if(listen(serv_sock, 5)==-1)
		error_handling("listen() error");

	epfd=epoll_create(EPOLL_SIZE);
	ep_events=malloc(sizeof(struct epoll_event)*EPOLL_SIZE);

	setnonblockingmode(serv_sock);
	event.events=EPOLLIN;
	event.data.fd=serv_sock;	
	epoll_ctl(epfd, EPOLL_CTL_ADD, serv_sock, &event);

	while(1)
	{
    
    
		event_cnt=epoll_wait(epfd, ep_events, EPOLL_SIZE, -1);
		if(event_cnt==-1)
		{
    
    
			puts("epoll_wait() error");
			break;
		}

		puts("return epoll_wait");
		for(i=0; i<event_cnt; i++)
		{
    
    
			if(ep_events[i].data.fd==serv_sock)
			{
    
    
				adr_sz=sizeof(clnt_adr);
				clnt_sock=accept(serv_sock, (struct sockaddr*)&clnt_adr, &adr_sz);
				setnonblockingmode(clnt_sock);
				event.events=EPOLLIN|EPOLLET;
				event.data.fd=clnt_sock;
				epoll_ctl(epfd, EPOLL_CTL_ADD, clnt_sock, &event);
				printf("connected client: %d \n", clnt_sock);
			}
			else
			{
    
    
					while(1)
					{
    
    
						str_len=read(ep_events[i].data.fd, buf, BUF_SIZE);
						if(str_len==0)    // close request!
						{
    
    
							epoll_ctl(epfd, EPOLL_CTL_DEL, ep_events[i].data.fd, NULL);
							close(ep_events[i].data.fd);
							printf("closed client: %d \n", ep_events[i].data.fd);
							break;
						}
						else if(str_len<0)
						{
    
    
							if(errno==EAGAIN)
								break;
						}
						else
						{
    
    
							write(ep_events[i].data.fd, buf, str_len);    // echo!
						}
				}
			}
		}
	}
	close(serv_sock);
	close(epfd);
	return 0;
}

void setnonblockingmode(int fd)
{
    
    
	int flag=fcntl(fd, F_GETFL, 0);
	fcntl(fd, F_SETFL, flag|O_NONBLOCK);
}
void error_handling(char *buf)
{
    
    
	fputs(buf, stderr);
	fputc('\n', stderr);
	exit(1);
}